2024-04-13T07:05:45.042 INFO:root:teuthology version: 0.0.1.dev267+g6c63784 2024-04-13T07:05:45.042 DEBUG:teuthology.run:Teuthology command: teuthology --description upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} --name teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi --archive /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941 --owner scheduled_teuthology@teuthology --verbose -- /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941/orig.config.yaml 2024-04-13T07:05:45.098 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-13T07:05:45.184 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941 branch: main description: upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} email: ceph-qa@ceph.com first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7654941' kernel: kdb: 1 sha1: distro ktype: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run basic import/export cli tests for rbd ' name: teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: ubuntu os_version: '22.04' overrides: admin_socket: branch: main ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 5f22a4974d6492df1d519913f5251731f043c582 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 5f22a4974d6492df1d519913f5251731f043c582 workunit: branch: main sha1: c2d8c3155d854ee204cb7c7261232571229d1d54 owner: scheduled_teuthology@teuthology priority: 850 repo: https://git.ceph.com/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' seed: 589 sha1: 5f22a4974d6492df1d519913f5251731f043c582 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: reef clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: reef clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' subset: 6/32 suite: upgrade suite_branch: main suite_path: /home/teuthworker/src/git.ceph.com_ceph_c2d8c3155d854ee204cb7c7261232571229d1d54/qa suite_relpath: qa suite_repo: https://git.ceph.com/ceph.git suite_sha1: c2d8c3155d854ee204cb7c7261232571229d1d54 targets: smithi089.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAdoi9yilb7HyWm2FrHTfsEG09R0IFB6Ej94+xwRwhbx2AiVgYGiop8OAHLq4Baq+VypOHRjG6a9C7q5xjKupVU= smithi203.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEf5PjW62a3Yg4cH5FlSOxYUIIBWGvr81e0xPm0qO3dzdkSlot2GYQ5p2a+3PExigHO9cupe0swhCx99GerfoWY= tasks: - install: branch: reef exclude_packages: - ceph-volume - cephadm: compiled_cephadm_branch: reef conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:reef - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 6c637841c215537a4502385240412f1966e0faab timestamp: 2024-04-13_03:08:05 tube: smithi user: teuthology verbose: false worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2226885 2024-04-13T07:05:45.184 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/git.ceph.com_ceph_c2d8c3155d854ee204cb7c7261232571229d1d54/qa; will attempt to use it 2024-04-13T07:05:45.185 INFO:teuthology.run:Found tasks at /home/teuthworker/src/git.ceph.com_ceph_c2d8c3155d854ee204cb7c7261232571229d1d54/qa/tasks 2024-04-13T07:05:45.185 INFO:teuthology.run_tasks:Running task internal.check_packages... 2024-04-13T07:05:45.187 INFO:teuthology.task.internal:Checking packages... 2024-04-13T07:05:45.209 INFO:teuthology.task.internal:Checking packages for os_type 'ubuntu', flavor 'default' and ceph hash '5f22a4974d6492df1d519913f5251731f043c582' 2024-04-13T07:05:45.209 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-13T07:05:45.209 INFO:teuthology.packaging:ref: None 2024-04-13T07:05:45.209 INFO:teuthology.packaging:tag: None 2024-04-13T07:05:45.209 INFO:teuthology.packaging:branch: main 2024-04-13T07:05:45.210 INFO:teuthology.packaging:sha1: 5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:05:45.210 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=main 2024-04-13T07:05:45.462 INFO:teuthology.task.internal:Found packages for ceph version 19.0.0-3080-gc2d8c315-1jammy 2024-04-13T07:05:45.466 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2024-04-13T07:05:45.473 INFO:teuthology.task.internal:no buildpackages task found 2024-04-13T07:05:45.473 INFO:teuthology.run_tasks:Running task internal.save_config... 2024-04-13T07:05:45.513 INFO:teuthology.task.internal:Saving configuration 2024-04-13T07:05:45.533 INFO:teuthology.run_tasks:Running task internal.check_lock... 2024-04-13T07:05:45.638 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-13T07:05:45.663 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi089.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-13 07:00:02.730171', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAILrSAamFRxT6s66oGWZKewrUg+ctjGNWyKtrurnZ0boE'} 2024-04-13T07:05:45.685 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi203.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-13 07:00:02.728712', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKWrtP2WBtQ4Pkp0klajwbVwP+ZOH3X85bhQZ/TeK0uR'} 2024-04-13T07:05:45.685 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2024-04-13T07:05:45.691 INFO:teuthology.task.internal:roles: ubuntu@smithi089.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2024-04-13T07:05:45.691 INFO:teuthology.task.internal:roles: ubuntu@smithi203.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2024-04-13T07:05:45.691 INFO:teuthology.run_tasks:Running task console_log... 2024-04-13T07:05:45.797 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7fb1b4094f70>, signals=[15]) 2024-04-13T07:05:45.797 INFO:teuthology.run_tasks:Running task internal.connect... 2024-04-13T07:05:45.804 INFO:teuthology.task.internal:Opening connections... 2024-04-13T07:05:45.804 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi089.front.sepia.ceph.com 2024-04-13T07:05:45.806 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:05:45.890 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi203.front.sepia.ceph.com 2024-04-13T07:05:45.891 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:05:45.968 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2024-04-13T07:05:45.975 DEBUG:teuthology.orchestra.run.smithi089:> uname -m 2024-04-13T07:05:45.980 INFO:teuthology.orchestra.run.smithi089.stdout:x86_64 2024-04-13T07:05:45.980 DEBUG:teuthology.orchestra.run.smithi089:> cat /etc/os-release 2024-04-13T07:05:46.033 INFO:teuthology.orchestra.run.smithi089.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-04-13T07:05:46.033 INFO:teuthology.orchestra.run.smithi089.stdout:NAME="Ubuntu" 2024-04-13T07:05:46.033 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION_ID="22.04" 2024-04-13T07:05:46.033 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-04-13T07:05:46.033 INFO:teuthology.orchestra.run.smithi089.stdout:VERSION_CODENAME=jammy 2024-04-13T07:05:46.033 INFO:teuthology.orchestra.run.smithi089.stdout:ID=ubuntu 2024-04-13T07:05:46.033 INFO:teuthology.orchestra.run.smithi089.stdout:ID_LIKE=debian 2024-04-13T07:05:46.034 INFO:teuthology.orchestra.run.smithi089.stdout:HOME_URL="https://www.ubuntu.com/" 2024-04-13T07:05:46.034 INFO:teuthology.orchestra.run.smithi089.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-04-13T07:05:46.034 INFO:teuthology.orchestra.run.smithi089.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-04-13T07:05:46.034 INFO:teuthology.orchestra.run.smithi089.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-04-13T07:05:46.034 INFO:teuthology.orchestra.run.smithi089.stdout:UBUNTU_CODENAME=jammy 2024-04-13T07:05:46.034 INFO:teuthology.lock.ops:Updating smithi089.front.sepia.ceph.com on lock server 2024-04-13T07:05:46.060 DEBUG:teuthology.orchestra.run.smithi203:> uname -m 2024-04-13T07:05:46.065 INFO:teuthology.orchestra.run.smithi203.stdout:x86_64 2024-04-13T07:05:46.066 DEBUG:teuthology.orchestra.run.smithi203:> cat /etc/os-release 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:PRETTY_NAME="Ubuntu 22.04.1 LTS" 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:NAME="Ubuntu" 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:VERSION_ID="22.04" 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:VERSION="22.04.1 LTS (Jammy Jellyfish)" 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:VERSION_CODENAME=jammy 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:ID=ubuntu 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:ID_LIKE=debian 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:HOME_URL="https://www.ubuntu.com/" 2024-04-13T07:05:46.111 INFO:teuthology.orchestra.run.smithi203.stdout:SUPPORT_URL="https://help.ubuntu.com/" 2024-04-13T07:05:46.112 INFO:teuthology.orchestra.run.smithi203.stdout:BUG_REPORT_URL="https://bugs.launchpad.net/ubuntu/" 2024-04-13T07:05:46.112 INFO:teuthology.orchestra.run.smithi203.stdout:PRIVACY_POLICY_URL="https://www.ubuntu.com/legal/terms-and-policies/privacy-policy" 2024-04-13T07:05:46.112 INFO:teuthology.orchestra.run.smithi203.stdout:UBUNTU_CODENAME=jammy 2024-04-13T07:05:46.112 INFO:teuthology.lock.ops:Updating smithi203.front.sepia.ceph.com on lock server 2024-04-13T07:05:46.248 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2024-04-13T07:05:46.258 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2024-04-13T07:05:46.285 INFO:teuthology.task.internal:Checking for old test directory... 2024-04-13T07:05:46.286 DEBUG:teuthology.orchestra.run.smithi089:> test '!' -e /home/ubuntu/cephtest 2024-04-13T07:05:46.288 DEBUG:teuthology.orchestra.run.smithi203:> test '!' -e /home/ubuntu/cephtest 2024-04-13T07:05:46.293 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2024-04-13T07:05:46.325 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2024-04-13T07:05:46.325 DEBUG:teuthology.orchestra.run.smithi089:> test -z $(ls -A /var/lib/ceph) 2024-04-13T07:05:46.334 DEBUG:teuthology.orchestra.run.smithi203:> test -z $(ls -A /var/lib/ceph) 2024-04-13T07:05:46.348 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2024-04-13T07:05:46.422 INFO:teuthology.run_tasks:Running task kernel... 2024-04-13T07:05:46.459 INFO:teuthology.task.kernel:normalize config orig: {'kdb': 1, 'sha1': 'distro'} 2024-04-13T07:05:46.460 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': 1, 'sha1': 'distro'}, 'mon.b': {'kdb': 1, 'sha1': 'distro'}}, timeout 300 2024-04-13T07:05:46.460 DEBUG:teuthology.orchestra.run.smithi089:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-13T07:05:46.460 DEBUG:teuthology.orchestra.run.smithi203:> test -f /run/.containerenv -o -f /.dockerenv 2024-04-13T07:05:46.465 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-13T07:05:46.465 DEBUG:teuthology.orchestra.run.smithi089:> uname -r 2024-04-13T07:05:46.466 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-13T07:05:46.466 DEBUG:teuthology.orchestra.run.smithi203:> uname -r 2024-04-13T07:05:46.512 INFO:teuthology.orchestra.run.smithi203.stdout:5.15.0-56-generic 2024-04-13T07:05:46.512 INFO:teuthology.task.kernel:Running kernel on smithi203: 5.15.0-56-generic 2024-04-13T07:05:46.512 DEBUG:teuthology.orchestra.run.smithi203:> sudo apt-get clean 2024-04-13T07:05:46.513 INFO:teuthology.orchestra.run.smithi089.stdout:5.15.0-56-generic 2024-04-13T07:05:46.513 INFO:teuthology.task.kernel:Running kernel on smithi089: 5.15.0-56-generic 2024-04-13T07:05:46.513 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get clean 2024-04-13T07:05:46.641 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2024-04-13T07:05:46.647 DEBUG:teuthology.orchestra.run.smithi203:> sudo apt-get update 2024-04-13T07:05:46.786 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-04-13T07:05:46.799 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-04-13T07:05:46.799 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB] 2024-04-13T07:05:46.799 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB] 2024-04-13T07:05:46.806 INFO:teuthology.orchestra.run.smithi203.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease [119 kB] 2024-04-13T07:05:46.874 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB] 2024-04-13T07:05:46.895 INFO:teuthology.orchestra.run.smithi203.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease [109 kB] 2024-04-13T07:05:46.898 INFO:teuthology.orchestra.run.smithi203.stdout:Get:4 http://security.ubuntu.com/ubuntu jammy-security InRelease [110 kB] 2024-04-13T07:05:47.008 INFO:teuthology.orchestra.run.smithi203.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [610 kB] 2024-04-13T07:05:47.027 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,343 kB] 2024-04-13T07:05:47.087 INFO:teuthology.orchestra.run.smithi203.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,562 kB] 2024-04-13T07:05:47.101 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 Packages [1,562 kB] 2024-04-13T07:05:47.126 INFO:teuthology.orchestra.run.smithi203.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [297 kB] 2024-04-13T07:05:47.131 INFO:teuthology.orchestra.run.smithi203.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [16.1 kB] 2024-04-13T07:05:47.131 INFO:teuthology.orchestra.run.smithi203.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [35.7 kB] 2024-04-13T07:05:47.132 INFO:teuthology.orchestra.run.smithi203.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,702 kB] 2024-04-13T07:05:47.141 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [444 kB] 2024-04-13T07:05:47.149 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [237 kB] 2024-04-13T07:05:47.153 INFO:teuthology.orchestra.run.smithi203.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [286 kB] 2024-04-13T07:05:47.154 INFO:teuthology.orchestra.run.smithi089.stdout:Get:9 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [11.4 kB] 2024-04-13T07:05:47.154 INFO:teuthology.orchestra.run.smithi089.stdout:Get:10 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,670 kB] 2024-04-13T07:05:47.158 INFO:teuthology.orchestra.run.smithi203.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [520 B] 2024-04-13T07:05:47.158 INFO:teuthology.orchestra.run.smithi203.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [699 kB] 2024-04-13T07:05:47.169 INFO:teuthology.orchestra.run.smithi203.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,074 kB] 2024-04-13T07:05:47.193 INFO:teuthology.orchestra.run.smithi089.stdout:Get:11 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [34.1 kB] 2024-04-13T07:05:47.193 INFO:teuthology.orchestra.run.smithi089.stdout:Get:12 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [280 kB] 2024-04-13T07:05:47.196 INFO:teuthology.orchestra.run.smithi089.stdout:Get:13 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [520 B] 2024-04-13T07:05:47.196 INFO:teuthology.orchestra.run.smithi089.stdout:Get:14 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [852 kB] 2024-04-13T07:05:47.208 INFO:teuthology.orchestra.run.smithi203.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [245 kB] 2024-04-13T07:05:47.208 INFO:teuthology.orchestra.run.smithi089.stdout:Get:15 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [600 kB] 2024-04-13T07:05:47.210 INFO:teuthology.orchestra.run.smithi203.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [22.1 kB] 2024-04-13T07:05:47.210 INFO:teuthology.orchestra.run.smithi203.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [49.7 kB] 2024-04-13T07:05:47.211 INFO:teuthology.orchestra.run.smithi203.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [6,028 B] 2024-04-13T07:05:47.211 INFO:teuthology.orchestra.run.smithi203.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [12.1 kB] 2024-04-13T07:05:47.211 INFO:teuthology.orchestra.run.smithi203.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [472 B] 2024-04-13T07:05:47.213 INFO:teuthology.orchestra.run.smithi089.stdout:Get:16 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [163 kB] 2024-04-13T07:05:47.214 INFO:teuthology.orchestra.run.smithi089.stdout:Get:17 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [16.8 kB] 2024-04-13T07:05:47.214 INFO:teuthology.orchestra.run.smithi089.stdout:Get:18 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-04-13T07:05:47.220 INFO:teuthology.orchestra.run.smithi203.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.2 kB] 2024-04-13T07:05:47.220 INFO:teuthology.orchestra.run.smithi203.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.1 kB] 2024-04-13T07:05:47.221 INFO:teuthology.orchestra.run.smithi203.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.0 kB] 2024-04-13T07:05:47.236 INFO:teuthology.orchestra.run.smithi089.stdout:Get:19 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-04-13T07:05:47.236 INFO:teuthology.orchestra.run.smithi089.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy-updates/main i386 Packages [610 kB] 2024-04-13T07:05:47.236 INFO:teuthology.orchestra.run.smithi089.stdout:Get:21 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-04-13T07:05:47.237 INFO:teuthology.orchestra.run.smithi089.stdout:Get:22 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [260 B] 2024-04-13T07:05:47.241 INFO:teuthology.orchestra.run.smithi203.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-04-13T07:05:47.241 INFO:teuthology.orchestra.run.smithi203.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.4 kB] 2024-04-13T07:05:47.241 INFO:teuthology.orchestra.run.smithi089.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy-updates/main Translation-en [297 kB] 2024-04-13T07:05:47.242 INFO:teuthology.orchestra.run.smithi203.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.2 kB] 2024-04-13T07:05:47.242 INFO:teuthology.orchestra.run.smithi203.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.2 kB] 2024-04-13T07:05:47.242 INFO:teuthology.orchestra.run.smithi203.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [644 B] 2024-04-13T07:05:47.243 INFO:teuthology.orchestra.run.smithi089.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 c-n-f Metadata [16.1 kB] 2024-04-13T07:05:47.243 INFO:teuthology.orchestra.run.smithi089.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 Packages [1,702 kB] 2024-04-13T07:05:47.269 INFO:teuthology.orchestra.run.smithi089.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/restricted i386 Packages [35.7 kB] 2024-04-13T07:05:47.269 INFO:teuthology.orchestra.run.smithi089.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/restricted Translation-en [286 kB] 2024-04-13T07:05:47.273 INFO:teuthology.orchestra.run.smithi089.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy-updates/restricted amd64 c-n-f Metadata [520 B] 2024-04-13T07:05:47.273 INFO:teuthology.orchestra.run.smithi089.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy-updates/universe i386 Packages [699 kB] 2024-04-13T07:05:47.282 INFO:teuthology.orchestra.run.smithi089.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 Packages [1,074 kB] 2024-04-13T07:05:47.294 INFO:teuthology.orchestra.run.smithi089.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy-updates/universe Translation-en [245 kB] 2024-04-13T07:05:47.296 INFO:teuthology.orchestra.run.smithi089.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 c-n-f Metadata [22.1 kB] 2024-04-13T07:05:47.297 INFO:teuthology.orchestra.run.smithi089.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse i386 Packages [6,028 B] 2024-04-13T07:05:47.297 INFO:teuthology.orchestra.run.smithi089.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 Packages [49.7 kB] 2024-04-13T07:05:47.311 INFO:teuthology.orchestra.run.smithi089.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse Translation-en [12.1 kB] 2024-04-13T07:05:47.311 INFO:teuthology.orchestra.run.smithi089.stdout:Get:36 http://archive.ubuntu.com/ubuntu jammy-updates/multiverse amd64 c-n-f Metadata [472 B] 2024-04-13T07:05:47.311 INFO:teuthology.orchestra.run.smithi089.stdout:Get:37 http://archive.ubuntu.com/ubuntu jammy-backports/main i386 Packages [59.2 kB] 2024-04-13T07:05:47.312 INFO:teuthology.orchestra.run.smithi089.stdout:Get:38 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 Packages [67.1 kB] 2024-04-13T07:05:47.313 INFO:teuthology.orchestra.run.smithi089.stdout:Get:39 http://archive.ubuntu.com/ubuntu jammy-backports/main Translation-en [11.0 kB] 2024-04-13T07:05:47.322 INFO:teuthology.orchestra.run.smithi203.stdout:Get:29 http://security.ubuntu.com/ubuntu jammy-security/main amd64 Packages [1,343 kB] 2024-04-13T07:05:47.332 INFO:teuthology.orchestra.run.smithi089.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy-backports/main amd64 c-n-f Metadata [388 B] 2024-04-13T07:05:47.332 INFO:teuthology.orchestra.run.smithi089.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 Packages [28.4 kB] 2024-04-13T07:05:47.333 INFO:teuthology.orchestra.run.smithi089.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy-backports/universe i386 Packages [17.2 kB] 2024-04-13T07:05:47.333 INFO:teuthology.orchestra.run.smithi089.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-backports/universe Translation-en [16.2 kB] 2024-04-13T07:05:47.333 INFO:teuthology.orchestra.run.smithi089.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy-backports/universe amd64 c-n-f Metadata [644 B] 2024-04-13T07:05:47.729 INFO:teuthology.orchestra.run.smithi203.stdout:Get:30 http://security.ubuntu.com/ubuntu jammy-security/main i386 Packages [444 kB] 2024-04-13T07:05:47.747 INFO:teuthology.orchestra.run.smithi203.stdout:Get:31 http://security.ubuntu.com/ubuntu jammy-security/main Translation-en [237 kB] 2024-04-13T07:05:47.756 INFO:teuthology.orchestra.run.smithi203.stdout:Get:32 http://security.ubuntu.com/ubuntu jammy-security/main amd64 c-n-f Metadata [11.4 kB] 2024-04-13T07:05:47.757 INFO:teuthology.orchestra.run.smithi203.stdout:Get:33 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 Packages [1,670 kB] 2024-04-13T07:05:47.848 INFO:teuthology.orchestra.run.smithi203.stdout:Get:34 http://security.ubuntu.com/ubuntu jammy-security/restricted i386 Packages [34.1 kB] 2024-04-13T07:05:47.849 INFO:teuthology.orchestra.run.smithi203.stdout:Get:35 http://security.ubuntu.com/ubuntu jammy-security/restricted Translation-en [280 kB] 2024-04-13T07:05:47.860 INFO:teuthology.orchestra.run.smithi203.stdout:Get:36 http://security.ubuntu.com/ubuntu jammy-security/restricted amd64 c-n-f Metadata [520 B] 2024-04-13T07:05:47.861 INFO:teuthology.orchestra.run.smithi203.stdout:Get:37 http://security.ubuntu.com/ubuntu jammy-security/universe i386 Packages [600 kB] 2024-04-13T07:05:47.881 INFO:teuthology.orchestra.run.smithi203.stdout:Get:38 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 Packages [852 kB] 2024-04-13T07:05:48.067 INFO:teuthology.orchestra.run.smithi203.stdout:Get:39 http://security.ubuntu.com/ubuntu jammy-security/universe Translation-en [163 kB] 2024-04-13T07:05:48.473 INFO:teuthology.orchestra.run.smithi203.stdout:Get:40 http://security.ubuntu.com/ubuntu jammy-security/universe amd64 c-n-f Metadata [16.8 kB] 2024-04-13T07:05:48.478 INFO:teuthology.orchestra.run.smithi203.stdout:Get:41 http://security.ubuntu.com/ubuntu jammy-security/multiverse i386 Packages [1,356 B] 2024-04-13T07:05:48.479 INFO:teuthology.orchestra.run.smithi203.stdout:Get:42 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 Packages [37.2 kB] 2024-04-13T07:05:48.492 INFO:teuthology.orchestra.run.smithi203.stdout:Get:43 http://security.ubuntu.com/ubuntu jammy-security/multiverse Translation-en [7,588 B] 2024-04-13T07:05:48.496 INFO:teuthology.orchestra.run.smithi203.stdout:Get:44 http://security.ubuntu.com/ubuntu jammy-security/multiverse amd64 c-n-f Metadata [260 B] 2024-04-13T07:05:52.708 INFO:teuthology.orchestra.run.smithi203.stdout:Fetched 12.9 MB in 2s (7,227 kB/s) 2024-04-13T07:05:52.820 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 12.9 MB in 2s (7,141 kB/s) 2024-04-13T07:05:54.089 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-04-13T07:05:54.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-04-13T07:05:54.140 INFO:teuthology.orchestra.run.smithi203.stdout:Reading package lists... 2024-04-13T07:05:54.156 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-04-13T07:05:54.158 DEBUG:teuthology.orchestra.run.smithi203:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-04-13T07:05:54.213 INFO:teuthology.orchestra.run.smithi203.stdout:Reading package lists... 2024-04-13T07:05:54.331 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2024-04-13T07:05:54.332 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2024-04-13T07:05:54.404 INFO:teuthology.orchestra.run.smithi203.stdout:Building dependency tree... 2024-04-13T07:05:54.405 INFO:teuthology.orchestra.run.smithi203.stdout:Reading state information... 2024-04-13T07:05:54.483 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages were automatically installed and are no longer required: 2024-04-13T07:05:54.483 INFO:teuthology.orchestra.run.smithi089.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-04-13T07:05:54.483 INFO:teuthology.orchestra.run.smithi089.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-04-13T07:05:54.483 INFO:teuthology.orchestra.run.smithi089.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-04-13T07:05:54.483 INFO:teuthology.orchestra.run.smithi089.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-04-13T07:05:54.484 INFO:teuthology.orchestra.run.smithi089.stdout: python2.7-minimal 2024-04-13T07:05:54.484 INFO:teuthology.orchestra.run.smithi089.stdout:Use 'sudo apt autoremove' to remove them. 2024-04-13T07:05:54.484 INFO:teuthology.orchestra.run.smithi089.stdout:The following additional packages will be installed: 2024-04-13T07:05:54.484 INFO:teuthology.orchestra.run.smithi089.stdout: linux-generic linux-headers-5.15.0-102 linux-headers-5.15.0-102-generic 2024-04-13T07:05:54.484 INFO:teuthology.orchestra.run.smithi089.stdout: linux-headers-generic linux-image-5.15.0-102-generic 2024-04-13T07:05:54.485 INFO:teuthology.orchestra.run.smithi089.stdout: linux-modules-5.15.0-102-generic linux-modules-extra-5.15.0-102-generic 2024-04-13T07:05:54.486 INFO:teuthology.orchestra.run.smithi089.stdout:Suggested packages: 2024-04-13T07:05:54.486 INFO:teuthology.orchestra.run.smithi089.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-04-13T07:05:54.486 INFO:teuthology.orchestra.run.smithi089.stdout:Recommended packages: 2024-04-13T07:05:54.486 INFO:teuthology.orchestra.run.smithi089.stdout: thermald 2024-04-13T07:05:54.541 INFO:teuthology.orchestra.run.smithi089.stdout:The following NEW packages will be installed: 2024-04-13T07:05:54.541 INFO:teuthology.orchestra.run.smithi089.stdout: linux-headers-5.15.0-102 linux-headers-5.15.0-102-generic 2024-04-13T07:05:54.541 INFO:teuthology.orchestra.run.smithi089.stdout: linux-image-5.15.0-102-generic linux-modules-5.15.0-102-generic 2024-04-13T07:05:54.541 INFO:teuthology.orchestra.run.smithi089.stdout: linux-modules-extra-5.15.0-102-generic 2024-04-13T07:05:54.543 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages will be upgraded: 2024-04-13T07:05:54.543 INFO:teuthology.orchestra.run.smithi089.stdout: linux-generic linux-headers-generic linux-image-generic 2024-04-13T07:05:54.557 INFO:teuthology.orchestra.run.smithi203.stdout:The following packages were automatically installed and are no longer required: 2024-04-13T07:05:54.557 INFO:teuthology.orchestra.run.smithi203.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-04-13T07:05:54.557 INFO:teuthology.orchestra.run.smithi203.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-04-13T07:05:54.557 INFO:teuthology.orchestra.run.smithi203.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-04-13T07:05:54.557 INFO:teuthology.orchestra.run.smithi203.stdout: python2 python2-dev python2-minimal python2.7 python2.7-dev 2024-04-13T07:05:54.557 INFO:teuthology.orchestra.run.smithi203.stdout: python2.7-minimal 2024-04-13T07:05:54.558 INFO:teuthology.orchestra.run.smithi203.stdout:Use 'sudo apt autoremove' to remove them. 2024-04-13T07:05:54.559 INFO:teuthology.orchestra.run.smithi203.stdout:The following additional packages will be installed: 2024-04-13T07:05:54.559 INFO:teuthology.orchestra.run.smithi203.stdout: linux-generic linux-headers-5.15.0-102 linux-headers-5.15.0-102-generic 2024-04-13T07:05:54.559 INFO:teuthology.orchestra.run.smithi203.stdout: linux-headers-generic linux-image-5.15.0-102-generic 2024-04-13T07:05:54.559 INFO:teuthology.orchestra.run.smithi203.stdout: linux-modules-5.15.0-102-generic linux-modules-extra-5.15.0-102-generic 2024-04-13T07:05:54.560 INFO:teuthology.orchestra.run.smithi203.stdout:Suggested packages: 2024-04-13T07:05:54.560 INFO:teuthology.orchestra.run.smithi203.stdout: fdutils linux-doc | linux-source-5.15.0 linux-tools 2024-04-13T07:05:54.560 INFO:teuthology.orchestra.run.smithi203.stdout:Recommended packages: 2024-04-13T07:05:54.560 INFO:teuthology.orchestra.run.smithi203.stdout: thermald 2024-04-13T07:05:54.612 INFO:teuthology.orchestra.run.smithi203.stdout:The following NEW packages will be installed: 2024-04-13T07:05:54.612 INFO:teuthology.orchestra.run.smithi203.stdout: linux-headers-5.15.0-102 linux-headers-5.15.0-102-generic 2024-04-13T07:05:54.612 INFO:teuthology.orchestra.run.smithi203.stdout: linux-image-5.15.0-102-generic linux-modules-5.15.0-102-generic 2024-04-13T07:05:54.612 INFO:teuthology.orchestra.run.smithi203.stdout: linux-modules-extra-5.15.0-102-generic 2024-04-13T07:05:54.613 INFO:teuthology.orchestra.run.smithi203.stdout:The following packages will be upgraded: 2024-04-13T07:05:54.614 INFO:teuthology.orchestra.run.smithi203.stdout: linux-generic linux-headers-generic linux-image-generic 2024-04-13T07:05:54.622 INFO:teuthology.orchestra.run.smithi089.stdout:3 upgraded, 5 newly installed, 0 to remove and 313 not upgraded. 2024-04-13T07:05:54.623 INFO:teuthology.orchestra.run.smithi089.stdout:Need to get 113 MB of archives. 2024-04-13T07:05:54.623 INFO:teuthology.orchestra.run.smithi089.stdout:After this operation, 583 MB of additional disk space will be used. 2024-04-13T07:05:54.623 INFO:teuthology.orchestra.run.smithi089.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-102-generic amd64 5.15.0-102.112 [22.7 MB] 2024-04-13T07:05:54.793 INFO:teuthology.orchestra.run.smithi203.stdout:3 upgraded, 5 newly installed, 0 to remove and 313 not upgraded. 2024-04-13T07:05:54.794 INFO:teuthology.orchestra.run.smithi203.stdout:Need to get 113 MB of archives. 2024-04-13T07:05:54.794 INFO:teuthology.orchestra.run.smithi203.stdout:After this operation, 583 MB of additional disk space will be used. 2024-04-13T07:05:54.794 INFO:teuthology.orchestra.run.smithi203.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-5.15.0-102-generic amd64 5.15.0-102.112 [22.7 MB] 2024-04-13T07:05:55.038 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-102-generic amd64 5.15.0-102.112 [11.5 MB] 2024-04-13T07:05:55.144 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-102-generic amd64 5.15.0-102.112 [63.9 MB] 2024-04-13T07:05:56.072 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.102.99 [1,700 B] 2024-04-13T07:05:56.072 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.102.99 [2,512 B] 2024-04-13T07:05:56.073 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-102 all 5.15.0-102.112 [12.4 MB] 2024-04-13T07:05:56.192 INFO:teuthology.orchestra.run.smithi203.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-5.15.0-102-generic amd64 5.15.0-102.112 [11.5 MB] 2024-04-13T07:05:56.216 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-102-generic amd64 5.15.0-102.112 [2,877 kB] 2024-04-13T07:05:56.244 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.102.99 [2,360 B] 2024-04-13T07:05:56.546 INFO:teuthology.orchestra.run.smithi203.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-modules-extra-5.15.0-102-generic amd64 5.15.0-102.112 [63.9 MB] 2024-04-13T07:05:56.698 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 113 MB in 2s (64.4 MB/s) 2024-04-13T07:05:56.870 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-modules-5.15.0-102-generic. 2024-04-13T07:05:58.778 INFO:teuthology.orchestra.run.smithi203.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-generic amd64 5.15.0.102.99 [1,700 B] 2024-04-13T07:05:58.779 INFO:teuthology.orchestra.run.smithi203.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-image-generic amd64 5.15.0.102.99 [2,512 B] 2024-04-13T07:05:58.779 INFO:teuthology.orchestra.run.smithi203.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-102 all 5.15.0-102.112 [12.4 MB] 2024-04-13T07:05:58.840 INFO:teuthology.orchestra.run.smithi089.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-04-13T07:05:58.844 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../0-linux-modules-5.15.0-102-generic_5.15.0-102.112_amd64.deb ... 2024-04-13T07:05:58.908 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-modules-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:05:59.266 INFO:teuthology.orchestra.run.smithi203.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-5.15.0-102-generic amd64 5.15.0-102.112 [2,877 kB] 2024-04-13T07:05:59.359 INFO:teuthology.orchestra.run.smithi203.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 linux-headers-generic amd64 5.15.0.102.99 [2,360 B] 2024-04-13T07:05:59.760 INFO:teuthology.orchestra.run.smithi203.stdout:Fetched 113 MB in 5s (23.9 MB/s) 2024-04-13T07:05:59.965 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package linux-modules-5.15.0-102-generic. 2024-04-13T07:06:00.960 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-image-5.15.0-102-generic. 2024-04-13T07:06:00.981 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../1-linux-image-5.15.0-102-generic_5.15.0-102.112_amd64.deb ... 2024-04-13T07:06:01.058 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-image-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:01.496 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-102-generic. 2024-04-13T07:06:01.517 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-102-generic_5.15.0-102.112_amd64.deb ... 2024-04-13T07:06:01.546 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-modules-extra-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:02.018 INFO:teuthology.orchestra.run.smithi203.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 129542 files and directories currently installed.) 2024-04-13T07:06:02.023 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../0-linux-modules-5.15.0-102-generic_5.15.0-102.112_amd64.deb ... 2024-04-13T07:06:02.137 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking linux-modules-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:04.246 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package linux-image-5.15.0-102-generic. 2024-04-13T07:06:04.268 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../1-linux-image-5.15.0-102-generic_5.15.0-102.112_amd64.deb ... 2024-04-13T07:06:04.396 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking linux-image-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:04.858 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package linux-modules-extra-5.15.0-102-generic. 2024-04-13T07:06:04.879 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../2-linux-modules-extra-5.15.0-102-generic_5.15.0-102.112_amd64.deb ... 2024-04-13T07:06:04.916 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking linux-modules-extra-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:07.697 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../3-linux-generic_5.15.0.102.99_amd64.deb ... 2024-04-13T07:06:07.816 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-generic (5.15.0.102.99) over (5.15.0.56.54) ... 2024-04-13T07:06:08.148 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.102.99_amd64.deb ... 2024-04-13T07:06:08.269 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-image-generic (5.15.0.102.99) over (5.15.0.56.54) ... 2024-04-13T07:06:08.539 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-headers-5.15.0-102. 2024-04-13T07:06:08.549 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../5-linux-headers-5.15.0-102_5.15.0-102.112_all.deb ... 2024-04-13T07:06:08.589 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-headers-5.15.0-102 (5.15.0-102.112) ... 2024-04-13T07:06:12.046 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../3-linux-generic_5.15.0.102.99_amd64.deb ... 2024-04-13T07:06:12.149 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking linux-generic (5.15.0.102.99) over (5.15.0.56.54) ... 2024-04-13T07:06:12.491 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../4-linux-image-generic_5.15.0.102.99_amd64.deb ... 2024-04-13T07:06:12.595 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking linux-image-generic (5.15.0.102.99) over (5.15.0.56.54) ... 2024-04-13T07:06:12.906 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package linux-headers-5.15.0-102. 2024-04-13T07:06:12.928 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../5-linux-headers-5.15.0-102_5.15.0-102.112_all.deb ... 2024-04-13T07:06:12.998 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking linux-headers-5.15.0-102 (5.15.0-102.112) ... 2024-04-13T07:06:14.479 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package linux-headers-5.15.0-102-generic. 2024-04-13T07:06:14.517 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../6-linux-headers-5.15.0-102-generic_5.15.0-102.112_amd64.deb ... 2024-04-13T07:06:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-headers-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:17.054 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.102.99_amd64.deb ... 2024-04-13T07:06:17.169 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking linux-headers-generic (5.15.0.102.99) over (5.15.0.56.54) ... 2024-04-13T07:06:17.469 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-headers-5.15.0-102 (5.15.0-102.112) ... 2024-04-13T07:06:17.590 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-headers-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:17.716 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-headers-generic (5.15.0.102.99) ... 2024-04-13T07:06:17.853 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-image-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:18.930 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package linux-headers-5.15.0-102-generic. 2024-04-13T07:06:18.966 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../6-linux-headers-5.15.0-102-generic_5.15.0-102.112_amd64.deb ... 2024-04-13T07:06:19.013 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking linux-headers-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:19.217 INFO:teuthology.orchestra.run.smithi089.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-04-13T07:06:19.217 INFO:teuthology.orchestra.run.smithi089.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-04-13T07:06:19.217 INFO:teuthology.orchestra.run.smithi089.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-102-generic 2024-04-13T07:06:19.217 INFO:teuthology.orchestra.run.smithi089.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-102-generic 2024-04-13T07:06:19.381 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-modules-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:20.775 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-modules-extra-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:21.672 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../7-linux-headers-generic_5.15.0.102.99_amd64.deb ... 2024-04-13T07:06:21.814 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking linux-headers-generic (5.15.0.102.99) over (5.15.0.56.54) ... 2024-04-13T07:06:22.160 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-image-generic (5.15.0.102.99) ... 2024-04-13T07:06:22.172 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up linux-headers-5.15.0-102 (5.15.0-102.112) ... 2024-04-13T07:06:22.853 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up linux-generic (5.15.0.102.99) ... 2024-04-13T07:06:23.148 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up linux-headers-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:23.162 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for linux-image-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:23.234 INFO:teuthology.orchestra.run.smithi089.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-04-13T07:06:23.234 INFO:teuthology.orchestra.run.smithi089.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-102-generic 2024-04-13T07:06:23.304 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up linux-headers-generic (5.15.0.102.99) ... 2024-04-13T07:06:23.433 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up linux-image-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:24.756 INFO:teuthology.orchestra.run.smithi203.stdout:I: /vmlinuz.old is now a symlink to boot/vmlinuz-5.15.0-56-generic 2024-04-13T07:06:24.756 INFO:teuthology.orchestra.run.smithi203.stdout:I: /initrd.img.old is now a symlink to boot/initrd.img-5.15.0-56-generic 2024-04-13T07:06:24.756 INFO:teuthology.orchestra.run.smithi203.stdout:I: /vmlinuz is now a symlink to boot/vmlinuz-5.15.0-102-generic 2024-04-13T07:06:24.756 INFO:teuthology.orchestra.run.smithi203.stdout:I: /initrd.img is now a symlink to boot/initrd.img-5.15.0-102-generic 2024-04-13T07:06:24.920 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up linux-modules-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:26.330 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up linux-modules-extra-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:27.731 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up linux-image-generic (5.15.0.102.99) ... 2024-04-13T07:06:27.857 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up linux-generic (5.15.0.102.99) ... 2024-04-13T07:06:27.984 INFO:teuthology.orchestra.run.smithi203.stdout:Processing triggers for linux-image-5.15.0-102-generic (5.15.0-102.112) ... 2024-04-13T07:06:28.062 INFO:teuthology.orchestra.run.smithi203.stdout:/etc/kernel/postinst.d/initramfs-tools: 2024-04-13T07:06:28.062 INFO:teuthology.orchestra.run.smithi203.stdout:update-initramfs: Generating /boot/initrd.img-5.15.0-102-generic 2024-04-13T07:06:37.801 INFO:teuthology.orchestra.run.smithi089.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-04-13T07:06:37.801 INFO:teuthology.orchestra.run.smithi089.stdout:Sourcing file `/etc/default/grub' 2024-04-13T07:06:37.817 INFO:teuthology.orchestra.run.smithi089.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-04-13T07:06:37.834 INFO:teuthology.orchestra.run.smithi089.stdout:Generating grub configuration file ... 2024-04-13T07:06:38.119 INFO:teuthology.orchestra.run.smithi089.stdout:Found linux image: /boot/vmlinuz-5.15.0-102-generic 2024-04-13T07:06:38.148 INFO:teuthology.orchestra.run.smithi089.stdout:Found initrd image: /boot/initrd.img-5.15.0-102-generic 2024-04-13T07:06:38.442 INFO:teuthology.orchestra.run.smithi089.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-04-13T07:06:38.448 INFO:teuthology.orchestra.run.smithi089.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-04-13T07:06:38.523 INFO:teuthology.orchestra.run.smithi089.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-04-13T07:06:38.529 INFO:teuthology.orchestra.run.smithi089.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-04-13T07:06:38.706 INFO:teuthology.orchestra.run.smithi089.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-04-13T07:06:38.706 INFO:teuthology.orchestra.run.smithi089.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-04-13T07:06:38.706 INFO:teuthology.orchestra.run.smithi089.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-04-13T07:06:38.749 INFO:teuthology.orchestra.run.smithi089.stdout:done 2024-04-13T07:06:39.220 DEBUG:teuthology.orchestra.run.smithi089:> dpkg -s linux-image-generic 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Package: linux-image-generic 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Status: install ok installed 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Priority: optional 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Section: kernel 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Installed-Size: 21 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Maintainer: Ubuntu Kernel Team 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Architecture: amd64 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Source: linux-meta 2024-04-13T07:06:39.239 INFO:teuthology.orchestra.run.smithi089.stdout:Version: 5.15.0.102.99 2024-04-13T07:06:39.240 INFO:teuthology.orchestra.run.smithi089.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-102), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3) 2024-04-13T07:06:39.240 INFO:teuthology.orchestra.run.smithi089.stdout:Depends: linux-image-5.15.0-102-generic, linux-modules-extra-5.15.0-102-generic, linux-firmware, intel-microcode, amd64-microcode 2024-04-13T07:06:39.240 INFO:teuthology.orchestra.run.smithi089.stdout:Recommends: thermald 2024-04-13T07:06:39.240 INFO:teuthology.orchestra.run.smithi089.stdout:Description: Generic Linux kernel image 2024-04-13T07:06:39.240 INFO:teuthology.orchestra.run.smithi089.stdout: This package will always depend on the latest generic kernel image 2024-04-13T07:06:39.240 INFO:teuthology.orchestra.run.smithi089.stdout: available. 2024-04-13T07:06:39.241 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-102-generic 2024-04-13T07:06:39.241 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-13T07:06:39.241 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-13T07:06:39.241 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2024-04-13T07:06:39.241 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi089.front.sepia.ceph.com, path=None, version=distro) 2024-04-13T07:06:39.241 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get clean 2024-04-13T07:06:39.374 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2024-04-13T07:06:39.489 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-04-13T07:06:39.500 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-04-13T07:06:39.521 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-04-13T07:06:39.602 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-04-13T07:06:40.780 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-04-13T07:06:40.798 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-04-13T07:06:40.857 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-04-13T07:06:41.054 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2024-04-13T07:06:41.054 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2024-04-13T07:06:41.223 INFO:teuthology.orchestra.run.smithi089.stdout:linux-image-generic is already the newest version (5.15.0.102.99). 2024-04-13T07:06:41.223 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages were automatically installed and are no longer required: 2024-04-13T07:06:41.223 INFO:teuthology.orchestra.run.smithi089.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-04-13T07:06:41.223 INFO:teuthology.orchestra.run.smithi089.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-04-13T07:06:41.224 INFO:teuthology.orchestra.run.smithi089.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-04-13T07:06:41.224 INFO:teuthology.orchestra.run.smithi089.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-04-13T07:06:41.224 INFO:teuthology.orchestra.run.smithi089.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-04-13T07:06:41.224 INFO:teuthology.orchestra.run.smithi089.stdout:Use 'sudo apt autoremove' to remove them. 2024-04-13T07:06:41.264 INFO:teuthology.orchestra.run.smithi089.stdout:0 upgraded, 0 newly installed, 0 to remove and 313 not upgraded. 2024-04-13T07:06:41.266 DEBUG:teuthology.orchestra.run.smithi089:> dpkg -s linux-image-generic 2024-04-13T07:06:41.277 INFO:teuthology.orchestra.run.smithi089.stdout:Package: linux-image-generic 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Status: install ok installed 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Priority: optional 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Section: kernel 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Installed-Size: 21 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Maintainer: Ubuntu Kernel Team 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Architecture: amd64 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Source: linux-meta 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Version: 5.15.0.102.99 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-102), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3) 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Depends: linux-image-5.15.0-102-generic, linux-modules-extra-5.15.0-102-generic, linux-firmware, intel-microcode, amd64-microcode 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Recommends: thermald 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout:Description: Generic Linux kernel image 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout: This package will always depend on the latest generic kernel image 2024-04-13T07:06:41.278 INFO:teuthology.orchestra.run.smithi089.stdout: available. 2024-04-13T07:06:41.279 DEBUG:teuthology.orchestra.run.smithi089:> mktemp 2024-04-13T07:06:41.322 INFO:teuthology.orchestra.run.smithi089.stdout:/tmp/tmp.k6jnTsYLFB 2024-04-13T07:06:41.323 DEBUG:teuthology.orchestra.run.smithi089:> sudo cp /boot/grub/grub.cfg /tmp/tmp.k6jnTsYLFB 2024-04-13T07:06:41.378 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 0666 /tmp/tmp.k6jnTsYLFB 2024-04-13T07:06:41.497 DEBUG:teuthology.orchestra.remote:smithi089:/tmp/tmp.k6jnTsYLFB is 10KB 2024-04-13T07:06:41.510 DEBUG:teuthology.orchestra.run.smithi089:> rm -fr /tmp/tmp.k6jnTsYLFB 2024-04-13T07:06:41.515 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-04-13T07:06:41.569 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:06:41.569 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-04-13T07:06:41.570 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-04-13T07:06:41.634 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-102-generic 2024-04-13T07:06:41.634 DEBUG:teuthology.orchestra.run.smithi089:> sudo update-grub 2024-04-13T07:06:42.568 INFO:teuthology.orchestra.run.smithi203.stdout:/etc/kernel/postinst.d/zz-update-grub: 2024-04-13T07:06:42.568 INFO:teuthology.orchestra.run.smithi203.stdout:Sourcing file `/etc/default/grub' 2024-04-13T07:06:42.581 INFO:teuthology.orchestra.run.smithi203.stdout:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-04-13T07:06:42.605 INFO:teuthology.orchestra.run.smithi203.stdout:Generating grub configuration file ... 2024-04-13T07:06:42.767 INFO:teuthology.orchestra.run.smithi089.stderr:Sourcing file `/etc/default/grub' 2024-04-13T07:06:42.768 INFO:teuthology.orchestra.run.smithi089.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-04-13T07:06:42.771 INFO:teuthology.orchestra.run.smithi089.stderr:Generating grub configuration file ... 2024-04-13T07:06:42.887 INFO:teuthology.orchestra.run.smithi203.stdout:Found linux image: /boot/vmlinuz-5.15.0-102-generic 2024-04-13T07:06:42.904 INFO:teuthology.orchestra.run.smithi203.stdout:Found initrd image: /boot/initrd.img-5.15.0-102-generic 2024-04-13T07:06:42.948 INFO:teuthology.orchestra.run.smithi089.stderr:Found linux image: /boot/vmlinuz-5.15.0-102-generic 2024-04-13T07:06:42.954 INFO:teuthology.orchestra.run.smithi089.stderr:Found initrd image: /boot/initrd.img-5.15.0-102-generic 2024-04-13T07:06:43.198 INFO:teuthology.orchestra.run.smithi089.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-04-13T07:06:43.203 INFO:teuthology.orchestra.run.smithi089.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-04-13T07:06:43.233 INFO:teuthology.orchestra.run.smithi203.stdout:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-04-13T07:06:43.239 INFO:teuthology.orchestra.run.smithi203.stdout:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-04-13T07:06:43.243 INFO:teuthology.orchestra.run.smithi089.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-04-13T07:06:43.249 INFO:teuthology.orchestra.run.smithi089.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-04-13T07:06:43.305 INFO:teuthology.orchestra.run.smithi203.stdout:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-04-13T07:06:43.310 INFO:teuthology.orchestra.run.smithi203.stdout:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-04-13T07:06:43.407 INFO:teuthology.orchestra.run.smithi089.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-04-13T07:06:43.407 INFO:teuthology.orchestra.run.smithi089.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-04-13T07:06:43.407 INFO:teuthology.orchestra.run.smithi089.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-04-13T07:06:43.419 INFO:teuthology.orchestra.run.smithi089.stderr:done 2024-04-13T07:06:43.433 DEBUG:teuthology.orchestra.run.smithi089:> sudo shutdown -r now 2024-04-13T07:06:43.469 INFO:teuthology.orchestra.run.smithi203.stdout:Warning: os-prober will not be executed to detect other bootable partitions. 2024-04-13T07:06:43.469 INFO:teuthology.orchestra.run.smithi203.stdout:Systems on them will not be added to the GRUB boot configuration. 2024-04-13T07:06:43.469 INFO:teuthology.orchestra.run.smithi203.stdout:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-04-13T07:06:43.507 INFO:teuthology.orchestra.run.smithi203.stdout:done 2024-04-13T07:06:44.040 DEBUG:teuthology.orchestra.run.smithi203:> dpkg -s linux-image-generic 2024-04-13T07:06:44.057 INFO:teuthology.orchestra.run.smithi203.stdout:Package: linux-image-generic 2024-04-13T07:06:44.057 INFO:teuthology.orchestra.run.smithi203.stdout:Status: install ok installed 2024-04-13T07:06:44.057 INFO:teuthology.orchestra.run.smithi203.stdout:Priority: optional 2024-04-13T07:06:44.057 INFO:teuthology.orchestra.run.smithi203.stdout:Section: kernel 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Installed-Size: 21 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Maintainer: Ubuntu Kernel Team 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Architecture: amd64 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Source: linux-meta 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Version: 5.15.0.102.99 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-102), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3) 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Depends: linux-image-5.15.0-102-generic, linux-modules-extra-5.15.0-102-generic, linux-firmware, intel-microcode, amd64-microcode 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Recommends: thermald 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout:Description: Generic Linux kernel image 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout: This package will always depend on the latest generic kernel image 2024-04-13T07:06:44.058 INFO:teuthology.orchestra.run.smithi203.stdout: available. 2024-04-13T07:06:44.058 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 5.15.0-56-generic Expected: 5.15.0-102-generic 2024-04-13T07:06:44.059 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2024-04-13T07:06:44.059 DEBUG:teuthology.task.kernel:src is distro, skipping download 2024-04-13T07:06:44.059 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2024-04-13T07:06:44.059 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi203.front.sepia.ceph.com, path=None, version=distro) 2024-04-13T07:06:44.059 DEBUG:teuthology.orchestra.run.smithi203:> sudo apt-get clean 2024-04-13T07:06:44.272 DEBUG:teuthology.orchestra.run.smithi203:> sudo apt-get update 2024-04-13T07:06:44.403 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-04-13T07:06:44.414 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:2 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-04-13T07:06:44.434 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-04-13T07:06:44.522 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:4 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-04-13T07:06:45.799 INFO:teuthology.orchestra.run.smithi203.stdout:Reading package lists... 2024-04-13T07:06:45.817 DEBUG:teuthology.orchestra.run.smithi203:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y install linux-image-generic 2024-04-13T07:06:45.875 INFO:teuthology.orchestra.run.smithi203.stdout:Reading package lists... 2024-04-13T07:06:46.068 INFO:teuthology.orchestra.run.smithi203.stdout:Building dependency tree... 2024-04-13T07:06:46.068 INFO:teuthology.orchestra.run.smithi203.stdout:Reading state information... 2024-04-13T07:06:46.244 INFO:teuthology.orchestra.run.smithi203.stdout:linux-image-generic is already the newest version (5.15.0.102.99). 2024-04-13T07:06:46.244 INFO:teuthology.orchestra.run.smithi203.stdout:The following packages were automatically installed and are no longer required: 2024-04-13T07:06:46.244 INFO:teuthology.orchestra.run.smithi203.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-04-13T07:06:46.244 INFO:teuthology.orchestra.run.smithi203.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-04-13T07:06:46.244 INFO:teuthology.orchestra.run.smithi203.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-04-13T07:06:46.245 INFO:teuthology.orchestra.run.smithi203.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-04-13T07:06:46.245 INFO:teuthology.orchestra.run.smithi203.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-04-13T07:06:46.245 INFO:teuthology.orchestra.run.smithi203.stdout:Use 'sudo apt autoremove' to remove them. 2024-04-13T07:06:46.286 INFO:teuthology.orchestra.run.smithi203.stdout:0 upgraded, 0 newly installed, 0 to remove and 313 not upgraded. 2024-04-13T07:06:46.288 DEBUG:teuthology.orchestra.run.smithi203:> dpkg -s linux-image-generic 2024-04-13T07:06:46.340 INFO:teuthology.orchestra.run.smithi203.stdout:Package: linux-image-generic 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Status: install ok installed 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Priority: optional 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Section: kernel 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Installed-Size: 21 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Maintainer: Ubuntu Kernel Team 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Architecture: amd64 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Source: linux-meta 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Version: 5.15.0.102.99 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Provides: spl-modules (= 2.1.5-1ubuntu6~22.04.3), v4l2loopback-modules (= 0.12.7-2ubuntu2~22.04.1), virtualbox-guest-modules (= 5.15.0-102), wireguard-modules (= 1.0.0), zfs-modules (= 2.1.5-1ubuntu6~22.04.3) 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Depends: linux-image-5.15.0-102-generic, linux-modules-extra-5.15.0-102-generic, linux-firmware, intel-microcode, amd64-microcode 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Recommends: thermald 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout:Description: Generic Linux kernel image 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout: This package will always depend on the latest generic kernel image 2024-04-13T07:06:46.341 INFO:teuthology.orchestra.run.smithi203.stdout: available. 2024-04-13T07:06:46.342 DEBUG:teuthology.orchestra.run.smithi203:> mktemp 2024-04-13T07:06:46.384 INFO:teuthology.orchestra.run.smithi203.stdout:/tmp/tmp.UoWCILlBjv 2024-04-13T07:06:46.384 DEBUG:teuthology.orchestra.run.smithi203:> sudo cp /boot/grub/grub.cfg /tmp/tmp.UoWCILlBjv 2024-04-13T07:06:46.437 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod 0666 /tmp/tmp.UoWCILlBjv 2024-04-13T07:06:46.562 DEBUG:teuthology.orchestra.remote:smithi203:/tmp/tmp.UoWCILlBjv is 10KB 2024-04-13T07:06:46.575 DEBUG:teuthology.orchestra.run.smithi203:> rm -fr /tmp/tmp.UoWCILlBjv 2024-04-13T07:06:46.581 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f -- /etc/grub.d/01_ceph_kernel 2024-04-13T07:06:46.633 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:06:46.633 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/grub.d/01_ceph_kernel 2024-04-13T07:06:46.633 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod 755 /etc/grub.d/01_ceph_kernel 2024-04-13T07:06:46.700 INFO:teuthology.task.kernel:Distro Kernel Version: 5.15.0-102-generic 2024-04-13T07:06:46.700 DEBUG:teuthology.orchestra.run.smithi203:> sudo update-grub 2024-04-13T07:06:47.909 INFO:teuthology.orchestra.run.smithi203.stderr:Sourcing file `/etc/default/grub' 2024-04-13T07:06:47.910 INFO:teuthology.orchestra.run.smithi203.stderr:Sourcing file `/etc/default/grub.d/init-select.cfg' 2024-04-13T07:06:47.915 INFO:teuthology.orchestra.run.smithi203.stderr:Generating grub configuration file ... 2024-04-13T07:06:48.099 INFO:teuthology.orchestra.run.smithi203.stderr:Found linux image: /boot/vmlinuz-5.15.0-102-generic 2024-04-13T07:06:48.104 INFO:teuthology.orchestra.run.smithi203.stderr:Found initrd image: /boot/initrd.img-5.15.0-102-generic 2024-04-13T07:06:48.336 INFO:teuthology.orchestra.run.smithi203.stderr:Found linux image: /boot/vmlinuz-5.15.0-56-generic 2024-04-13T07:06:48.341 INFO:teuthology.orchestra.run.smithi203.stderr:Found initrd image: /boot/initrd.img-5.15.0-56-generic 2024-04-13T07:06:48.385 INFO:teuthology.orchestra.run.smithi203.stderr:Found linux image: /boot/vmlinuz-5.4.0-135-generic 2024-04-13T07:06:48.389 INFO:teuthology.orchestra.run.smithi203.stderr:Found initrd image: /boot/initrd.img-5.4.0-135-generic 2024-04-13T07:06:48.507 INFO:teuthology.orchestra.run.smithi203.stderr:Warning: os-prober will not be executed to detect other bootable partitions. 2024-04-13T07:06:48.507 INFO:teuthology.orchestra.run.smithi203.stderr:Systems on them will not be added to the GRUB boot configuration. 2024-04-13T07:06:48.507 INFO:teuthology.orchestra.run.smithi203.stderr:Check GRUB_DISABLE_OS_PROBER documentation entry. 2024-04-13T07:06:48.530 INFO:teuthology.orchestra.run.smithi203.stderr:done 2024-04-13T07:06:48.539 DEBUG:teuthology.orchestra.run.smithi203:> sudo shutdown -r now 2024-04-13T07:07:13.457 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2024-04-13T07:07:13.457 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-04-13T07:07:13.459 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:07:18.546 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2024-04-13T07:07:18.546 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-04-13T07:07:18.547 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:07:31.999 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-04-13T07:07:36.862 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.203 2024-04-13T07:07:41.004 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-04-13T07:07:41.004 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:07:44.062 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-04-13T07:07:45.864 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-04-13T07:07:45.865 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:07:48.926 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.203 2024-04-13T07:07:56.066 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-04-13T07:07:56.067 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:08:00.930 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-04-13T07:08:00.931 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:08:14.494 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.89 2024-04-13T07:08:19.358 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.203 2024-04-13T07:08:29.500 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-04-13T07:08:29.502 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:08:30.267 DEBUG:teuthology.orchestra.run.smithi089:> true 2024-04-13T07:08:30.969 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-04-13T07:08:30.969 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "5.15.0-102-generic"... 2024-04-13T07:08:30.970 DEBUG:teuthology.orchestra.run.smithi089:> uname -r 2024-04-13T07:08:31.018 INFO:teuthology.orchestra.run.smithi089.stdout:5.15.0-102-generic 2024-04-13T07:08:31.019 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-102-generic vs 5.15.0-102-generic 2024-04-13T07:08:31.019 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-13T07:08:31.019 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-04-13T07:08:32.020 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2024-04-13T07:08:32.020 DEBUG:teuthology.orchestra.run.smithi089:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-13T07:08:32.114 INFO:teuthology.orchestra.run.smithi089.stdout:ttyS1 2024-04-13T07:08:32.144 DEBUG:teuthology.parallel:result is None 2024-04-13T07:08:34.359 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-04-13T07:08:34.360 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:08:34.362 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.203 2024-04-13T07:08:52.364 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-04-13T07:08:52.365 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:08:52.661 DEBUG:teuthology.orchestra.run.smithi203:> true 2024-04-13T07:08:53.224 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-04-13T07:08:53.224 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "5.15.0-102-generic"... 2024-04-13T07:08:53.224 DEBUG:teuthology.orchestra.run.smithi203:> uname -r 2024-04-13T07:08:53.269 INFO:teuthology.orchestra.run.smithi203.stdout:5.15.0-102-generic 2024-04-13T07:08:53.269 DEBUG:teuthology.task.kernel:current kernel version is 5.15.0-102-generic vs 5.15.0-102-generic 2024-04-13T07:08:53.269 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2024-04-13T07:08:53.269 DEBUG:teuthology.task.kernel:Distro of this test job: ubuntu 2024-04-13T07:08:54.270 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2024-04-13T07:08:54.271 DEBUG:teuthology.orchestra.run.smithi203:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2024-04-13T07:08:54.360 INFO:teuthology.orchestra.run.smithi203.stdout:ttyS1 2024-04-13T07:08:54.391 DEBUG:teuthology.parallel:result is None 2024-04-13T07:08:54.391 INFO:teuthology.run_tasks:Running task internal.base... 2024-04-13T07:08:54.399 INFO:teuthology.task.internal:Creating test directory... 2024-04-13T07:08:54.399 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-13T07:08:54.402 DEBUG:teuthology.orchestra.run.smithi203:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2024-04-13T07:08:54.408 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2024-04-13T07:08:54.437 INFO:teuthology.run_tasks:Running task internal.archive... 2024-04-13T07:08:54.458 INFO:teuthology.task.internal:Creating archive directory... 2024-04-13T07:08:54.458 DEBUG:teuthology.orchestra.run.smithi089:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-13T07:08:54.461 DEBUG:teuthology.orchestra.run.smithi203:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2024-04-13T07:08:54.493 INFO:teuthology.run_tasks:Running task internal.coredump... 2024-04-13T07:08:54.506 INFO:teuthology.task.internal:Enabling coredump saving... 2024-04-13T07:08:54.506 DEBUG:teuthology.orchestra.run.smithi089:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-13T07:08:54.531 DEBUG:teuthology.orchestra.run.smithi203:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2024-04-13T07:08:54.547 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-13T07:08:54.555 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-13T07:08:54.557 INFO:teuthology.orchestra.run.smithi203.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-13T07:08:54.565 INFO:teuthology.orchestra.run.smithi203.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2024-04-13T07:08:54.567 INFO:teuthology.run_tasks:Running task internal.sudo... 2024-04-13T07:08:54.574 INFO:teuthology.task.internal:Configuring sudo... 2024-04-13T07:08:54.574 DEBUG:teuthology.orchestra.run.smithi089:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-13T07:08:54.602 DEBUG:teuthology.orchestra.run.smithi203:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2024-04-13T07:08:54.641 INFO:teuthology.run_tasks:Running task internal.syslog... 2024-04-13T07:08:54.651 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2024-04-13T07:08:54.651 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-13T07:08:54.660 DEBUG:teuthology.orchestra.run.smithi203:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2024-04-13T07:08:54.690 DEBUG:teuthology.orchestra.run.smithi089:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-13T07:08:54.710 DEBUG:teuthology.orchestra.run.smithi089:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-13T07:08:54.758 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:08:54.758 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-13T07:08:54.818 DEBUG:teuthology.orchestra.run.smithi203:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2024-04-13T07:08:54.824 DEBUG:teuthology.orchestra.run.smithi203:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2024-04-13T07:08:54.873 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:08:54.874 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2024-04-13T07:08:54.936 DEBUG:teuthology.orchestra.run.smithi089:> sudo service rsyslog restart 2024-04-13T07:08:54.939 DEBUG:teuthology.orchestra.run.smithi203:> sudo service rsyslog restart 2024-04-13T07:08:55.016 INFO:teuthology.run_tasks:Running task internal.timer... 2024-04-13T07:08:55.024 INFO:teuthology.task.internal:Starting timer... 2024-04-13T07:08:55.025 INFO:teuthology.run_tasks:Running task pcp... 2024-04-13T07:08:55.076 INFO:teuthology.run_tasks:Running task selinux... 2024-04-13T07:08:55.111 DEBUG:teuthology.task.selinux:Excluding smithi089: OS 'ubuntu' does not support SELinux 2024-04-13T07:08:55.111 DEBUG:teuthology.task.selinux:Excluding smithi203: OS 'ubuntu' does not support SELinux 2024-04-13T07:08:55.112 DEBUG:teuthology.task.selinux:Getting current SELinux state 2024-04-13T07:08:55.112 DEBUG:teuthology.task.selinux:Existing SELinux modes: {} 2024-04-13T07:08:55.112 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2024-04-13T07:08:55.112 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2024-04-13T07:08:55.187 DEBUG:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2024-04-13T07:08:55.204 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2024-04-13T07:08:55.225 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi089.front.sepia.ceph.com,smithi203.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2024-04-13T07:15:54.390 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi089.front.sepia.ceph.com'), Remote(name='ubuntu@smithi203.front.sepia.ceph.com')] 2024-04-13T07:15:54.412 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-04-13T07:15:54.414 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi089.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:15:54.494 DEBUG:teuthology.orchestra.run.smithi089:> true 2024-04-13T07:15:54.570 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi089.front.sepia.ceph.com' 2024-04-13T07:15:54.571 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-04-13T07:15:54.571 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi203.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2024-04-13T07:15:54.651 DEBUG:teuthology.orchestra.run.smithi203:> true 2024-04-13T07:15:54.722 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi203.front.sepia.ceph.com' 2024-04-13T07:15:54.723 INFO:teuthology.run_tasks:Running task clock... 2024-04-13T07:15:54.735 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2024-04-13T07:15:54.736 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-13T07:15:54.736 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-13T07:15:54.738 INFO:teuthology.orchestra.run:Running command with timeout 360 2024-04-13T07:15:54.738 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-13T07:15:54.766 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-04-13T07:15:54.766 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Command line: ntpd -gq 2024-04-13T07:15:54.767 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: ---------------------------------------------------- 2024-04-13T07:15:54.767 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: ntp-4 is maintained by Network Time Foundation, 2024-04-13T07:15:54.767 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-04-13T07:15:54.767 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: corporation. Support and training for ntp-4 are 2024-04-13T07:15:54.767 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: available at https://www.nwtime.org/support 2024-04-13T07:15:54.767 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: ---------------------------------------------------- 2024-04-13T07:15:54.768 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: proto: precision = 0.061 usec (-24) 2024-04-13T07:15:54.768 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: basedate set to 2022-02-04 2024-04-13T07:15:54.768 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: gps base set to 2022-02-06 (week 2196) 2024-04-13T07:15:54.768 INFO:teuthology.orchestra.run.smithi089.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-04-13T07:15:54.768 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-04-13T07:15:54.769 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: restrict ::: KOD does nothing without LIMITED. 2024-04-13T07:15:54.769 INFO:teuthology.orchestra.run.smithi089.stderr:restrict ::: KOD does nothing without LIMITED. 2024-04-13T07:15:54.770 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Listen and drop on 0 v6wildcard [::]:123 2024-04-13T07:15:54.770 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-04-13T07:15:54.770 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Listen normally on 2 lo 127.0.0.1:123 2024-04-13T07:15:54.770 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Listen normally on 3 enp3s0f1 172.21.15.89:123 2024-04-13T07:15:54.770 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Listen normally on 4 lo [::1]:123 2024-04-13T07:15:54.770 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:d501%5]:123 2024-04-13T07:15:54.770 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:15:54 ntpd[17193]: Listening on routing socket on fd #22 for interface updates 2024-04-13T07:15:54.789 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: ntpd 4.2.8p15@1.3728-o Wed Feb 16 17:13:02 UTC 2022 (1): Starting 2024-04-13T07:15:54.789 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Command line: ntpd -gq 2024-04-13T07:15:54.789 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: ---------------------------------------------------- 2024-04-13T07:15:54.789 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: ntp-4 is maintained by Network Time Foundation, 2024-04-13T07:15:54.789 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Inc. (NTF), a non-profit 501(c)(3) public-benefit 2024-04-13T07:15:54.790 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: corporation. Support and training for ntp-4 are 2024-04-13T07:15:54.790 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: available at https://www.nwtime.org/support 2024-04-13T07:15:54.790 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: ---------------------------------------------------- 2024-04-13T07:15:54.790 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: proto: precision = 0.060 usec (-24) 2024-04-13T07:15:54.790 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: basedate set to 2022-02-04 2024-04-13T07:15:54.790 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: gps base set to 2022-02-06 (week 2196) 2024-04-13T07:15:54.791 INFO:teuthology.orchestra.run.smithi203.stderr:restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-04-13T07:15:54.791 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: restrict 0.0.0.0: KOD does nothing without LIMITED. 2024-04-13T07:15:54.791 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: restrict ::: KOD does nothing without LIMITED. 2024-04-13T07:15:54.791 INFO:teuthology.orchestra.run.smithi203.stderr:restrict ::: KOD does nothing without LIMITED. 2024-04-13T07:15:54.792 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Listen and drop on 0 v6wildcard [::]:123 2024-04-13T07:15:54.792 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Listen and drop on 1 v4wildcard 0.0.0.0:123 2024-04-13T07:15:54.793 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Listen normally on 2 lo 127.0.0.1:123 2024-04-13T07:15:54.793 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Listen normally on 3 enp3s0f1 172.21.15.203:123 2024-04-13T07:15:54.793 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Listen normally on 4 lo [::1]:123 2024-04-13T07:15:54.793 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Listen normally on 5 enp3s0f1 [fe80::ec4:7aff:fe8f:4a23%5]:123 2024-04-13T07:15:54.793 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:15:54 ntpd[17171]: Listening on routing socket on fd #22 for interface updates 2024-04-13T07:15:55.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:55 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:55.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:55 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:55.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:55 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:55.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:55 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:55.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:55 ntpd[17193]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:55.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:55 ntpd[17193]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:55.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:55.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:55.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:55.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:55.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:55.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:55.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:55.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:55.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:55.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:55.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:55.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:55 ntpd[17171]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:56.789 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:56 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:56.789 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:56 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:57.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:57 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:57.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:57 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:57.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:57 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:57.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:57 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:57.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:57 ntpd[17193]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:57.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:57 ntpd[17193]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:57.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:57.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:57.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:57 ntpd[17171]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:58.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:58 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:58.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:58 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:58.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:58 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:58.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:58 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:58.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:58 ntpd[17193]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:58.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:58 ntpd[17193]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:59.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:59.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:15:59 ntpd[17193]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:59.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:59.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:59.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:15:59.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:15:59.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:15:59.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:15:59.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:15:59 ntpd[17171]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:16:00.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:00 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:00.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:00 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:00.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:00 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:00.767 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:00 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:00.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:00 ntpd[17193]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:16:00.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:00 ntpd[17193]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:16:01.768 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:01.769 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:16:01.770 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:16:01.770 INFO:teuthology.orchestra.run.smithi089.stdout:13 Apr 07:16:01 ntpd[17193]: ntpd: time slew +0.000018 s 2024-04-13T07:16:01.770 INFO:teuthology.orchestra.run.smithi089.stdout:ntpd: time slew +0.000018s 2024-04-13T07:16:01.771 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-04-13T07:16:01.771 INFO:teuthology.orchestra.run.smithi089.stderr:13 Apr 07:16:01 ntpd[17193]: can't open /var/log/ntpstats/loopstats.20240413: Permission denied 2024-04-13T07:16:01.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:01.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:01.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:01.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:01.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:01.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:01.790 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: couldn't unlink /var/log/ntpstats/rawstats: Permission denied 2024-04-13T07:16:01.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: can't open /var/log/ntpstats/rawstats.20240413: Permission denied 2024-04-13T07:16:01.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: couldn't unlink /var/log/ntpstats/peerstats: Permission denied 2024-04-13T07:16:01.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: can't open /var/log/ntpstats/peerstats.20240413: Permission denied 2024-04-13T07:16:01.791 INFO:teuthology.orchestra.run.smithi203.stdout:13 Apr 07:16:01 ntpd[17171]: ntpd: time slew -0.001953 s 2024-04-13T07:16:01.791 INFO:teuthology.orchestra.run.smithi203.stdout:ntpd: time slew -0.001953s 2024-04-13T07:16:01.791 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: couldn't unlink /var/log/ntpstats/loopstats: Permission denied 2024-04-13T07:16:01.792 INFO:teuthology.orchestra.run.smithi203.stderr:13 Apr 07:16:01 ntpd[17171]: can't open /var/log/ntpstats/loopstats.20240413: Permission denied 2024-04-13T07:16:01.824 INFO:teuthology.orchestra.run.smithi089.stdout: remote refid st t when poll reach delay offset jitter 2024-04-13T07:16:01.824 INFO:teuthology.orchestra.run.smithi089.stdout:============================================================================== 2024-04-13T07:16:01.825 INFO:teuthology.orchestra.run.smithi089.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-04-13T07:16:01.825 INFO:teuthology.orchestra.run.smithi089.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-04-13T07:16:01.825 INFO:teuthology.orchestra.run.smithi089.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-04-13T07:16:01.825 INFO:teuthology.orchestra.run.smithi089.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-04-13T07:16:01.836 INFO:teuthology.orchestra.run.smithi203.stdout: remote refid st t when poll reach delay offset jitter 2024-04-13T07:16:01.836 INFO:teuthology.orchestra.run.smithi203.stdout:============================================================================== 2024-04-13T07:16:01.836 INFO:teuthology.orchestra.run.smithi203.stdout: hv01.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-04-13T07:16:01.836 INFO:teuthology.orchestra.run.smithi203.stdout: hv02.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-04-13T07:16:01.836 INFO:teuthology.orchestra.run.smithi203.stdout: hv03.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-04-13T07:16:01.836 INFO:teuthology.orchestra.run.smithi203.stdout: hv04.front.sepi .INIT. 16 u - 64 0 0.000 +0.000 0.000 2024-04-13T07:16:01.837 INFO:teuthology.run_tasks:Running task install... 2024-04-13T07:16:01.846 DEBUG:teuthology.task.install:project ceph 2024-04-13T07:16:01.846 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '5f22a4974d6492df1d519913f5251731f043c582'}} 2024-04-13T07:16:01.846 DEBUG:teuthology.task.install:config {'branch': 'reef', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '5f22a4974d6492df1d519913f5251731f043c582'} 2024-04-13T07:16:01.846 INFO:teuthology.task.install:Using flavor: default 2024-04-13T07:16:01.852 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2024-04-13T07:16:01.852 INFO:teuthology.task.install:extra packages: [] 2024-04-13T07:16:01.853 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-key list | grep Ceph 2024-04-13T07:16:01.853 DEBUG:teuthology.orchestra.run.smithi203:> sudo apt-key list | grep Ceph 2024-04-13T07:16:01.921 INFO:teuthology.orchestra.run.smithi089.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-04-13T07:16:01.942 INFO:teuthology.orchestra.run.smithi203.stderr:Warning: apt-key is deprecated. Manage keyring files in trusted.gpg.d instead (see apt-key(8)). 2024-04-13T07:16:01.947 INFO:teuthology.orchestra.run.smithi089.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-04-13T07:16:01.947 INFO:teuthology.orchestra.run.smithi089.stdout:uid [ unknown] Ceph.com (release key) 2024-04-13T07:16:01.948 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-04-13T07:16:01.948 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-13T07:16:01.948 INFO:teuthology.packaging:ref: None 2024-04-13T07:16:01.948 INFO:teuthology.packaging:tag: None 2024-04-13T07:16:01.948 INFO:teuthology.packaging:branch: reef 2024-04-13T07:16:01.949 INFO:teuthology.packaging:sha1: 5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:16:01.949 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-04-13T07:16:01.971 INFO:teuthology.orchestra.run.smithi203.stdout:uid [ unknown] Ceph automated package build (Ceph automated package build) 2024-04-13T07:16:01.971 INFO:teuthology.orchestra.run.smithi203.stdout:uid [ unknown] Ceph.com (release key) 2024-04-13T07:16:01.971 INFO:teuthology.task.install.deb:Installing packages: ceph, cephadm, ceph-mds, ceph-mgr, ceph-common, ceph-fuse, ceph-test, radosgw, python3-rados, python3-rgw, python3-cephfs, python3-rbd, libcephfs2, libcephfs-dev, librados2, librbd1, rbd-fuse on remote deb x86_64 2024-04-13T07:16:01.971 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-13T07:16:01.971 INFO:teuthology.packaging:ref: None 2024-04-13T07:16:01.972 INFO:teuthology.packaging:tag: None 2024-04-13T07:16:01.972 INFO:teuthology.packaging:branch: reef 2024-04-13T07:16:01.972 INFO:teuthology.packaging:sha1: 5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:16:01.972 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-04-13T07:16:02.282 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default/ 2024-04-13T07:16:02.283 INFO:teuthology.task.install.deb:Package version is 18.2.2-1217-g7d36322c-1jammy 2024-04-13T07:16:02.401 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:16:02.401 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-04-13T07:16:02.415 DEBUG:teuthology.orchestra.run.smithi203:> sudo apt-get update 2024-04-13T07:16:02.493 INFO:teuthology.task.install.deb:Pulling from https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default/ 2024-04-13T07:16:02.494 INFO:teuthology.task.install.deb:Package version is 18.2.2-1217-g7d36322c-1jammy 2024-04-13T07:16:02.609 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:16:02.609 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/apt/sources.list.d/ceph.list 2024-04-13T07:16:02.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo apt-get update 2024-04-13T07:16:02.630 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-04-13T07:16:02.631 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-04-13T07:16:02.646 INFO:teuthology.orchestra.run.smithi203.stdout:Ign:3 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy InRelease 2024-04-13T07:16:02.676 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-04-13T07:16:02.683 INFO:teuthology.orchestra.run.smithi203.stdout:Get:5 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy Release [7,662 B] 2024-04-13T07:16:02.705 INFO:teuthology.orchestra.run.smithi203.stdout:Ign:6 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy Release.gpg 2024-04-13T07:16:02.724 INFO:teuthology.orchestra.run.smithi203.stdout:Hit:7 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-04-13T07:16:02.730 INFO:teuthology.orchestra.run.smithi203.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,416 B] 2024-04-13T07:16:02.768 INFO:teuthology.orchestra.run.smithi203.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.5 kB] 2024-04-13T07:16:02.840 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:1 http://archive.ubuntu.com/ubuntu jammy InRelease 2024-04-13T07:16:02.840 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:2 http://security.ubuntu.com/ubuntu jammy-security InRelease 2024-04-13T07:16:02.881 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:3 http://archive.ubuntu.com/ubuntu jammy-updates InRelease 2024-04-13T07:16:02.931 INFO:teuthology.orchestra.run.smithi089.stdout:Hit:4 http://archive.ubuntu.com/ubuntu jammy-backports InRelease 2024-04-13T07:16:02.991 INFO:teuthology.orchestra.run.smithi203.stdout:Fetched 28.6 kB in 0s (71.8 kB/s) 2024-04-13T07:16:03.004 INFO:teuthology.orchestra.run.smithi089.stdout:Ign:5 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy InRelease 2024-04-13T07:16:03.068 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy Release [7,662 B] 2024-04-13T07:16:03.130 INFO:teuthology.orchestra.run.smithi089.stdout:Ign:7 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy Release.gpg 2024-04-13T07:16:03.187 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main i386 Packages [3,416 B] 2024-04-13T07:16:03.261 INFO:teuthology.orchestra.run.smithi089.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 Packages [17.5 kB] 2024-04-13T07:16:03.348 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 28.6 kB in 1s (53.4 kB/s) 2024-04-13T07:16:03.946 INFO:teuthology.orchestra.run.smithi203.stdout:Reading package lists... 2024-04-13T07:16:03.962 DEBUG:teuthology.orchestra.run.smithi203:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=18.2.2-1217-g7d36322c-1jammy cephadm=18.2.2-1217-g7d36322c-1jammy ceph-mds=18.2.2-1217-g7d36322c-1jammy ceph-mgr=18.2.2-1217-g7d36322c-1jammy ceph-common=18.2.2-1217-g7d36322c-1jammy ceph-fuse=18.2.2-1217-g7d36322c-1jammy ceph-test=18.2.2-1217-g7d36322c-1jammy radosgw=18.2.2-1217-g7d36322c-1jammy python3-rados=18.2.2-1217-g7d36322c-1jammy python3-rgw=18.2.2-1217-g7d36322c-1jammy python3-cephfs=18.2.2-1217-g7d36322c-1jammy python3-rbd=18.2.2-1217-g7d36322c-1jammy libcephfs2=18.2.2-1217-g7d36322c-1jammy libcephfs-dev=18.2.2-1217-g7d36322c-1jammy librados2=18.2.2-1217-g7d36322c-1jammy librbd1=18.2.2-1217-g7d36322c-1jammy rbd-fuse=18.2.2-1217-g7d36322c-1jammy 2024-04-13T07:16:04.025 INFO:teuthology.orchestra.run.smithi203.stdout:Reading package lists... 2024-04-13T07:16:04.221 INFO:teuthology.orchestra.run.smithi203.stdout:Building dependency tree... 2024-04-13T07:16:04.222 INFO:teuthology.orchestra.run.smithi203.stdout:Reading state information... 2024-04-13T07:16:04.351 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-04-13T07:16:04.368 DEBUG:teuthology.orchestra.run.smithi089:> sudo DEBIAN_FRONTEND=noninteractive apt-get -y --force-yes -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" install ceph=18.2.2-1217-g7d36322c-1jammy cephadm=18.2.2-1217-g7d36322c-1jammy ceph-mds=18.2.2-1217-g7d36322c-1jammy ceph-mgr=18.2.2-1217-g7d36322c-1jammy ceph-common=18.2.2-1217-g7d36322c-1jammy ceph-fuse=18.2.2-1217-g7d36322c-1jammy ceph-test=18.2.2-1217-g7d36322c-1jammy radosgw=18.2.2-1217-g7d36322c-1jammy python3-rados=18.2.2-1217-g7d36322c-1jammy python3-rgw=18.2.2-1217-g7d36322c-1jammy python3-cephfs=18.2.2-1217-g7d36322c-1jammy python3-rbd=18.2.2-1217-g7d36322c-1jammy libcephfs2=18.2.2-1217-g7d36322c-1jammy libcephfs-dev=18.2.2-1217-g7d36322c-1jammy librados2=18.2.2-1217-g7d36322c-1jammy librbd1=18.2.2-1217-g7d36322c-1jammy rbd-fuse=18.2.2-1217-g7d36322c-1jammy 2024-04-13T07:16:04.416 INFO:teuthology.orchestra.run.smithi203.stdout:The following packages were automatically installed and are no longer required: 2024-04-13T07:16:04.416 INFO:teuthology.orchestra.run.smithi203.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-04-13T07:16:04.416 INFO:teuthology.orchestra.run.smithi203.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-04-13T07:16:04.417 INFO:teuthology.orchestra.run.smithi203.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-04-13T07:16:04.417 INFO:teuthology.orchestra.run.smithi203.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-04-13T07:16:04.417 INFO:teuthology.orchestra.run.smithi203.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-04-13T07:16:04.417 INFO:teuthology.orchestra.run.smithi203.stdout:Use 'sudo apt autoremove' to remove them. 2024-04-13T07:16:04.418 INFO:teuthology.orchestra.run.smithi203.stdout:The following additional packages will be installed: 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: python3-asyncssh python3-babel python3-bcrypt python3-cachetools 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-04-13T07:16:04.419 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pygments python3-pyinotify python3-pytest python3-repoze.lru 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-requests python3-requests-oauthlib python3-routes python3-rsa 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-04-13T07:16:04.420 INFO:teuthology.orchestra.run.smithi203.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-04-13T07:16:04.421 INFO:teuthology.orchestra.run.smithi203.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-04-13T07:16:04.421 INFO:teuthology.orchestra.run.smithi203.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-04-13T07:16:04.421 INFO:teuthology.orchestra.run.smithi203.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout:Suggested packages: 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout: python3-watchdog 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout:Recommended packages: 2024-04-13T07:16:04.422 INFO:teuthology.orchestra.run.smithi203.stdout: btrfs-tools 2024-04-13T07:16:04.430 INFO:teuthology.orchestra.run.smithi089.stdout:Reading package lists... 2024-04-13T07:16:04.520 INFO:teuthology.orchestra.run.smithi203.stdout:The following NEW packages will be installed: 2024-04-13T07:16:04.521 INFO:teuthology.orchestra.run.smithi203.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-04-13T07:16:04.521 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-04-13T07:16:04.521 INFO:teuthology.orchestra.run.smithi203.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-04-13T07:16:04.521 INFO:teuthology.orchestra.run.smithi203.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-04-13T07:16:04.521 INFO:teuthology.orchestra.run.smithi203.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-04-13T07:16:04.522 INFO:teuthology.orchestra.run.smithi203.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-04-13T07:16:04.522 INFO:teuthology.orchestra.run.smithi203.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python-babel-localedata 2024-04-13T07:16:04.522 INFO:teuthology.orchestra.run.smithi203.stdout: python-pastedeploy-tpl python3-asyncssh python3-babel python3-bcrypt 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-certifi python3-cheroot python3-cherrypy3 python3-google-auth 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-idna python3-iniconfig python3-jaraco.classes 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-jinja2 python3-joblib python3-kubernetes python3-logutils 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-mako python3-markupsafe python3-natsort python3-openssl 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pyasn1 python3-pyasn1-modules python3-pygments python3-pyinotify 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-pytest python3-rados python3-rbd python3-repoze.lru python3-requests 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2024-04-13T07:16:04.523 INFO:teuthology.orchestra.run.smithi203.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-04-13T07:16:04.524 INFO:teuthology.orchestra.run.smithi203.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-04-13T07:16:04.524 INFO:teuthology.orchestra.run.smithi203.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-04-13T07:16:04.524 INFO:teuthology.orchestra.run.smithi203.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-04-13T07:16:04.524 INFO:teuthology.orchestra.run.smithi203.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2024-04-13T07:16:04.524 INFO:teuthology.orchestra.run.smithi203.stdout: socat xmlstarlet 2024-04-13T07:16:04.627 INFO:teuthology.orchestra.run.smithi089.stdout:Building dependency tree... 2024-04-13T07:16:04.627 INFO:teuthology.orchestra.run.smithi089.stdout:Reading state information... 2024-04-13T07:16:04.669 INFO:teuthology.orchestra.run.smithi203.stdout:0 upgraded, 109 newly installed, 0 to remove and 300 not upgraded. 2024-04-13T07:16:04.669 INFO:teuthology.orchestra.run.smithi203.stdout:Need to get 162 MB of archives. 2024-04-13T07:16:04.669 INFO:teuthology.orchestra.run.smithi203.stdout:After this operation, 743 MB of additional disk space will be used. 2024-04-13T07:16:04.669 INFO:teuthology.orchestra.run.smithi203.stdout:Get:1 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 18.2.2-1217-g7d36322c-1jammy [3,440 kB] 2024-04-13T07:16:04.713 INFO:teuthology.orchestra.run.smithi203.stdout:Get:2 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-04-13T07:16:04.821 INFO:teuthology.orchestra.run.smithi089.stdout:The following packages were automatically installed and are no longer required: 2024-04-13T07:16:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: libboost-iostreams1.74.0 libboost-thread1.74.0 libgfapi0 libgfrpc0 libgfxdr0 2024-04-13T07:16:04.821 INFO:teuthology.orchestra.run.smithi089.stdout: libglusterfs0 libiscsi7 libpmemobj1 libpython2-dev libpython2-stdlib 2024-04-13T07:16:04.822 INFO:teuthology.orchestra.run.smithi089.stdout: libpython2.7 libpython2.7-dev libpython2.7-minimal libpython2.7-stdlib 2024-04-13T07:16:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: linux-image-5.4.0-135-generic linux-modules-5.4.0-135-generic python2 2024-04-13T07:16:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: python2-dev python2-minimal python2.7 python2.7-dev python2.7-minimal 2024-04-13T07:16:04.825 INFO:teuthology.orchestra.run.smithi089.stdout:Use 'sudo apt autoremove' to remove them. 2024-04-13T07:16:04.825 INFO:teuthology.orchestra.run.smithi089.stdout:The following additional packages will be installed: 2024-04-13T07:16:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-base ceph-mgr-cephadm ceph-mgr-dashboard ceph-mgr-diskprediction-local 2024-04-13T07:16:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-k8sevents ceph-mgr-modules-core ceph-mon ceph-osd ceph-volume jq 2024-04-13T07:16:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 liboath0 libonig5 2024-04-13T07:16:04.825 INFO:teuthology.orchestra.run.smithi089.stdout: libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 libradosstriper1 2024-04-13T07:16:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: librdkafka1 librgw2 libsqlite3-mod-ceph libthrift-0.16.0 nvme-cli 2024-04-13T07:16:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: python-asyncssh-doc python-babel-localedata python-pastedeploy-tpl 2024-04-13T07:16:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: python3-asyncssh python3-babel python3-bcrypt python3-cachetools 2024-04-13T07:16:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: python3-ceph-argparse python3-ceph-common python3-certifi python3-cheroot 2024-04-13T07:16:04.826 INFO:teuthology.orchestra.run.smithi089.stdout: python3-cherrypy3 python3-google-auth python3-idna python3-iniconfig 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-jaraco.classes python3-jaraco.collections python3-jaraco.functools 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-jaraco.text python3-jinja2 python3-joblib python3-kubernetes 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-logutils python3-mako python3-markupsafe python3-natsort 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-openssl python3-paste python3-pastedeploy python3-pastescript 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pecan python3-pluggy python3-portend python3-prettytable 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-psutil python3-py python3-pyasn1 python3-pyasn1-modules 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pygments python3-pyinotify python3-pytest python3-repoze.lru 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-requests python3-requests-oauthlib python3-routes python3-rsa 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-04-13T07:16:04.827 INFO:teuthology.orchestra.run.smithi089.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-04-13T07:16:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-04-13T07:16:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n socat xmlstarlet 2024-04-13T07:16:04.828 INFO:teuthology.orchestra.run.smithi089.stdout:Suggested packages: 2024-04-13T07:16:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: python3-influxdb python-jinja2-doc python3-beaker python-mako-doc 2024-04-13T07:16:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: python-natsort-doc python-openssl-doc python3-openssl-dbg httpd-wsgi 2024-04-13T07:16:04.828 INFO:teuthology.orchestra.run.smithi089.stdout: libapache2-mod-python libapache2-mod-scgi libjs-mochikit python-pecan-doc 2024-04-13T07:16:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: python-psutil-doc subversion python-pygments-doc python-pyinotify-doc 2024-04-13T07:16:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: python3-socks python-requests-doc python3-dap python-sklearn-doc ipython3 2024-04-13T07:16:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: python-waitress-doc python-webob-doc python-webtest-doc python-werkzeug-doc 2024-04-13T07:16:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: python3-watchdog 2024-04-13T07:16:04.829 INFO:teuthology.orchestra.run.smithi089.stdout:Recommended packages: 2024-04-13T07:16:04.829 INFO:teuthology.orchestra.run.smithi089.stdout: btrfs-tools 2024-04-13T07:16:04.896 INFO:teuthology.orchestra.run.smithi203.stdout:Get:3 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 18.2.2-1217-g7d36322c-1jammy [3,105 kB] 2024-04-13T07:16:04.924 INFO:teuthology.orchestra.run.smithi089.stdout:The following NEW packages will be installed: 2024-04-13T07:16:04.924 INFO:teuthology.orchestra.run.smithi089.stdout: ceph ceph-base ceph-common ceph-fuse ceph-mds ceph-mgr ceph-mgr-cephadm 2024-04-13T07:16:04.924 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-dashboard ceph-mgr-diskprediction-local ceph-mgr-k8sevents 2024-04-13T07:16:04.926 INFO:teuthology.orchestra.run.smithi089.stdout: ceph-mgr-modules-core ceph-mon ceph-osd ceph-test ceph-volume cephadm jq 2024-04-13T07:16:04.926 INFO:teuthology.orchestra.run.smithi089.stdout: libcephfs-dev libcephfs2 libdouble-conversion3 libfmt8 libjq1 liblttng-ust1 2024-04-13T07:16:04.926 INFO:teuthology.orchestra.run.smithi089.stdout: liboath0 libonig5 libpcre2-16-0 libqt5core5a libqt5dbus5 libqt5network5 2024-04-13T07:16:04.926 INFO:teuthology.orchestra.run.smithi089.stdout: librados2 libradosstriper1 librbd1 librdkafka1 librgw2 libsqlite3-mod-ceph 2024-04-13T07:16:04.926 INFO:teuthology.orchestra.run.smithi089.stdout: libthrift-0.16.0 nvme-cli python-asyncssh-doc python-babel-localedata 2024-04-13T07:16:04.926 INFO:teuthology.orchestra.run.smithi089.stdout: python-pastedeploy-tpl python3-asyncssh python3-babel python3-bcrypt 2024-04-13T07:16:04.926 INFO:teuthology.orchestra.run.smithi089.stdout: python3-cachetools python3-ceph-argparse python3-ceph-common python3-cephfs 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-certifi python3-cheroot python3-cherrypy3 python3-google-auth 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-idna python3-iniconfig python3-jaraco.classes 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-jaraco.collections python3-jaraco.functools python3-jaraco.text 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-jinja2 python3-joblib python3-kubernetes python3-logutils 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-mako python3-markupsafe python3-natsort python3-openssl 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-paste python3-pastedeploy python3-pastescript python3-pecan 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pluggy python3-portend python3-prettytable python3-psutil python3-py 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pyasn1 python3-pyasn1-modules python3-pygments python3-pyinotify 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-pytest python3-rados python3-rbd python3-repoze.lru python3-requests 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-requests-oauthlib python3-rgw python3-routes python3-rsa 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-simplegeneric python3-simplejson python3-singledispatch 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-sklearn python3-sklearn-lib python3-tempita python3-tempora 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-threadpoolctl python3-toml python3-urllib3 python3-waitress 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-wcwidth python3-webob python3-websocket python3-webtest 2024-04-13T07:16:04.927 INFO:teuthology.orchestra.run.smithi089.stdout: python3-werkzeug python3-zc.lockfile qttranslations5-l10n radosgw rbd-fuse 2024-04-13T07:16:04.928 INFO:teuthology.orchestra.run.smithi089.stdout: socat xmlstarlet 2024-04-13T07:16:04.989 INFO:teuthology.orchestra.run.smithi203.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-04-13T07:16:05.001 INFO:teuthology.orchestra.run.smithi203.stdout:Get:5 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 18.2.2-1217-g7d36322c-1jammy [680 kB] 2024-04-13T07:16:05.022 INFO:teuthology.orchestra.run.smithi203.stdout:Get:6 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 18.2.2-1217-g7d36322c-1jammy [356 kB] 2024-04-13T07:16:05.034 INFO:teuthology.orchestra.run.smithi203.stdout:Get:7 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 18.2.2-1217-g7d36322c-1jammy [32.8 kB] 2024-04-13T07:16:05.035 INFO:teuthology.orchestra.run.smithi203.stdout:Get:8 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 18.2.2-1217-g7d36322c-1jammy [180 kB] 2024-04-13T07:16:05.041 INFO:teuthology.orchestra.run.smithi203.stdout:Get:9 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 18.2.2-1217-g7d36322c-1jammy [63.7 kB] 2024-04-13T07:16:05.043 INFO:teuthology.orchestra.run.smithi203.stdout:Get:10 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 18.2.2-1217-g7d36322c-1jammy [330 kB] 2024-04-13T07:16:05.060 INFO:teuthology.orchestra.run.smithi203.stdout:Get:11 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 18.2.2-1217-g7d36322c-1jammy [5,891 kB] 2024-04-13T07:16:05.111 INFO:teuthology.orchestra.run.smithi089.stdout:0 upgraded, 109 newly installed, 0 to remove and 300 not upgraded. 2024-04-13T07:16:05.111 INFO:teuthology.orchestra.run.smithi089.stdout:Need to get 162 MB of archives. 2024-04-13T07:16:05.111 INFO:teuthology.orchestra.run.smithi089.stdout:After this operation, 743 MB of additional disk space will be used. 2024-04-13T07:16:05.111 INFO:teuthology.orchestra.run.smithi089.stdout:Get:1 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libfmt8 amd64 8.1.1+ds1-2 [60.2 kB] 2024-04-13T07:16:05.122 INFO:teuthology.orchestra.run.smithi089.stdout:Get:2 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 librados2 amd64 18.2.2-1217-g7d36322c-1jammy [3,440 kB] 2024-04-13T07:16:05.162 INFO:teuthology.orchestra.run.smithi203.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-04-13T07:16:05.176 INFO:teuthology.orchestra.run.smithi203.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-04-13T07:16:05.248 INFO:teuthology.orchestra.run.smithi203.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-04-13T07:16:05.255 INFO:teuthology.orchestra.run.smithi203.stdout:Get:15 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 18.2.2-1217-g7d36322c-1jammy [112 kB] 2024-04-13T07:16:05.258 INFO:teuthology.orchestra.run.smithi203.stdout:Get:16 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 18.2.2-1217-g7d36322c-1jammy [441 kB] 2024-04-13T07:16:05.274 INFO:teuthology.orchestra.run.smithi203.stdout:Get:17 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 18.2.2-1217-g7d36322c-1jammy [21.9 MB] 2024-04-13T07:16:05.395 INFO:teuthology.orchestra.run.smithi089.stdout:Get:3 http://archive.ubuntu.com/ubuntu jammy/main amd64 liblttng-ust1 amd64 2.13.1-1ubuntu1 [190 kB] 2024-04-13T07:16:05.442 INFO:teuthology.orchestra.run.smithi203.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-04-13T07:16:05.452 INFO:teuthology.orchestra.run.smithi203.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-04-13T07:16:05.481 INFO:teuthology.orchestra.run.smithi203.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-04-13T07:16:05.489 INFO:teuthology.orchestra.run.smithi203.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-04-13T07:16:05.490 INFO:teuthology.orchestra.run.smithi203.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-04-13T07:16:05.491 INFO:teuthology.orchestra.run.smithi203.stdout:Get:23 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-04-13T07:16:05.567 INFO:teuthology.orchestra.run.smithi089.stdout:Get:4 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libdouble-conversion3 amd64 3.1.7-4 [39.0 kB] 2024-04-13T07:16:05.581 INFO:teuthology.orchestra.run.smithi089.stdout:Get:5 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 libpcre2-16-0 amd64 10.39-3ubuntu0.1 [203 kB] 2024-04-13T07:16:05.654 INFO:teuthology.orchestra.run.smithi089.stdout:Get:6 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5core5a amd64 5.15.3+dfsg-2ubuntu0.2 [2,006 kB] 2024-04-13T07:16:05.673 INFO:teuthology.orchestra.run.smithi203.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-04-13T07:16:05.849 INFO:teuthology.orchestra.run.smithi089.stdout:Get:7 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5dbus5 amd64 5.15.3+dfsg-2ubuntu0.2 [222 kB] 2024-04-13T07:16:05.859 INFO:teuthology.orchestra.run.smithi089.stdout:Get:8 http://archive.ubuntu.com/ubuntu jammy-updates/universe amd64 libqt5network5 amd64 5.15.3+dfsg-2ubuntu0.2 [731 kB] 2024-04-13T07:16:05.885 INFO:teuthology.orchestra.run.smithi203.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-04-13T07:16:05.887 INFO:teuthology.orchestra.run.smithi089.stdout:Get:9 http://archive.ubuntu.com/ubuntu jammy/universe amd64 libthrift-0.16.0 amd64 0.16.0-2 [267 kB] 2024-04-13T07:16:05.896 INFO:teuthology.orchestra.run.smithi089.stdout:Get:10 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-wcwidth all 0.2.5+dfsg1-1 [21.9 kB] 2024-04-13T07:16:05.897 INFO:teuthology.orchestra.run.smithi089.stdout:Get:11 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-prettytable all 2.5.0-2 [31.3 kB] 2024-04-13T07:16:05.899 INFO:teuthology.orchestra.run.smithi089.stdout:Get:12 http://archive.ubuntu.com/ubuntu jammy/universe amd64 librdkafka1 amd64 1.8.0-1build1 [633 kB] 2024-04-13T07:16:05.899 INFO:teuthology.orchestra.run.smithi203.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-04-13T07:16:06.017 INFO:teuthology.orchestra.run.smithi203.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-04-13T07:16:06.025 INFO:teuthology.orchestra.run.smithi203.stdout:Get:28 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-04-13T07:16:06.036 INFO:teuthology.orchestra.run.smithi203.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-04-13T07:16:06.079 INFO:teuthology.orchestra.run.smithi089.stdout:Get:13 http://archive.ubuntu.com/ubuntu jammy/main amd64 liboath0 amd64 2.6.7-3build1 [41.1 kB] 2024-04-13T07:16:06.126 INFO:teuthology.orchestra.run.smithi203.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-04-13T07:16:06.127 INFO:teuthology.orchestra.run.smithi203.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-04-13T07:16:06.127 INFO:teuthology.orchestra.run.smithi203.stdout:Get:32 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-04-13T07:16:06.127 INFO:teuthology.orchestra.run.smithi203.stdout:Get:33 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 18.2.2-1217-g7d36322c-1jammy [5,002 kB] 2024-04-13T07:16:06.128 INFO:teuthology.orchestra.run.smithi203.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-04-13T07:16:06.196 INFO:teuthology.orchestra.run.smithi203.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-04-13T07:16:06.221 INFO:teuthology.orchestra.run.smithi203.stdout:Get:36 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 18.2.2-1217-g7d36322c-1jammy [241 kB] 2024-04-13T07:16:06.233 INFO:teuthology.orchestra.run.smithi203.stdout:Get:37 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 18.2.2-1217-g7d36322c-1jammy [130 kB] 2024-04-13T07:16:06.239 INFO:teuthology.orchestra.run.smithi203.stdout:Get:38 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 18.2.2-1217-g7d36322c-1jammy [1,490 kB] 2024-04-13T07:16:06.294 INFO:teuthology.orchestra.run.smithi203.stdout:Get:39 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 18.2.2-1217-g7d36322c-1jammy [6,079 kB] 2024-04-13T07:16:06.327 INFO:teuthology.orchestra.run.smithi089.stdout:Get:14 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.functools all 3.4.0-2 [9,030 B] 2024-04-13T07:16:06.335 INFO:teuthology.orchestra.run.smithi089.stdout:Get:15 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-cheroot all 8.5.2+ds1-1ubuntu3.1 [71.1 kB] 2024-04-13T07:16:06.369 INFO:teuthology.orchestra.run.smithi203.stdout:Get:40 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-04-13T07:16:06.433 INFO:teuthology.orchestra.run.smithi089.stdout:Get:16 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.classes all 3.2.1-3 [6,452 B] 2024-04-13T07:16:06.441 INFO:teuthology.orchestra.run.smithi089.stdout:Get:17 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.text all 3.6.0-2 [8,716 B] 2024-04-13T07:16:06.450 INFO:teuthology.orchestra.run.smithi089.stdout:Get:18 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-jaraco.collections all 3.4.0-2 [11.4 kB] 2024-04-13T07:16:06.463 INFO:teuthology.orchestra.run.smithi089.stdout:Get:19 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempora all 4.1.2-1 [14.8 kB] 2024-04-13T07:16:06.476 INFO:teuthology.orchestra.run.smithi089.stdout:Get:20 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-portend all 3.0.0-1 [7,240 B] 2024-04-13T07:16:06.483 INFO:teuthology.orchestra.run.smithi089.stdout:Get:21 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-zc.lockfile all 2.0-1 [8,980 B] 2024-04-13T07:16:06.490 INFO:teuthology.orchestra.run.smithi089.stdout:Get:22 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cherrypy3 all 18.6.1-4 [208 kB] 2024-04-13T07:16:06.518 INFO:teuthology.orchestra.run.smithi203.stdout:Get:41 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 18.2.2-1217-g7d36322c-1jammy [22.3 MB] 2024-04-13T07:16:06.536 INFO:teuthology.orchestra.run.smithi203.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-04-13T07:16:06.561 INFO:teuthology.orchestra.run.smithi203.stdout:Get:43 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-04-13T07:16:06.577 INFO:teuthology.orchestra.run.smithi089.stdout:Get:23 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 librbd1 amd64 18.2.2-1217-g7d36322c-1jammy [3,105 kB] 2024-04-13T07:16:06.588 INFO:teuthology.orchestra.run.smithi089.stdout:Get:24 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-natsort all 8.0.2-1 [35.3 kB] 2024-04-13T07:16:06.691 INFO:teuthology.orchestra.run.smithi203.stdout:Get:44 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-04-13T07:16:06.710 INFO:teuthology.orchestra.run.smithi203.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-04-13T07:16:06.717 INFO:teuthology.orchestra.run.smithi203.stdout:Get:46 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-04-13T07:16:06.763 INFO:teuthology.orchestra.run.smithi089.stdout:Get:25 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-logutils all 0.3.3-8 [17.6 kB] 2024-04-13T07:16:06.788 INFO:teuthology.orchestra.run.smithi203.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-04-13T07:16:06.817 INFO:teuthology.orchestra.run.smithi203.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-04-13T07:16:06.823 INFO:teuthology.orchestra.run.smithi203.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-04-13T07:16:06.931 INFO:teuthology.orchestra.run.smithi089.stdout:Get:26 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-markupsafe amd64 2.0.1-2build1 [12.7 kB] 2024-04-13T07:16:06.946 INFO:teuthology.orchestra.run.smithi203.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-04-13T07:16:06.946 INFO:teuthology.orchestra.run.smithi203.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-04-13T07:16:06.955 INFO:teuthology.orchestra.run.smithi089.stdout:Get:27 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-mako all 1.1.3+ds1-2ubuntu0.1 [60.5 kB] 2024-04-13T07:16:07.042 INFO:teuthology.orchestra.run.smithi089.stdout:Get:28 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs2 amd64 18.2.2-1217-g7d36322c-1jammy [680 kB] 2024-04-13T07:16:07.086 INFO:teuthology.orchestra.run.smithi089.stdout:Get:29 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplegeneric all 0.8.1-3 [11.3 kB] 2024-04-13T07:16:07.104 INFO:teuthology.orchestra.run.smithi089.stdout:Get:30 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-singledispatch all 3.4.0.3-3 [7,320 B] 2024-04-13T07:16:07.110 INFO:teuthology.orchestra.run.smithi089.stdout:Get:31 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webob all 1:1.8.6-1.1 [85.2 kB] 2024-04-13T07:16:07.115 INFO:teuthology.orchestra.run.smithi203.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-04-13T07:16:07.178 INFO:teuthology.orchestra.run.smithi089.stdout:Get:32 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-rados amd64 18.2.2-1217-g7d36322c-1jammy [356 kB] 2024-04-13T07:16:07.182 INFO:teuthology.orchestra.run.smithi089.stdout:Get:33 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-waitress all 1.4.4-1.1ubuntu1 [47.3 kB] 2024-04-13T07:16:07.209 INFO:teuthology.orchestra.run.smithi089.stdout:Get:34 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-tempita all 0.5.2-6ubuntu1 [15.1 kB] 2024-04-13T07:16:07.216 INFO:teuthology.orchestra.run.smithi089.stdout:Get:35 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-paste all 3.5.0+dfsg1-1 [456 kB] 2024-04-13T07:16:07.241 INFO:teuthology.orchestra.run.smithi089.stdout:Get:36 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-argparse all 18.2.2-1217-g7d36322c-1jammy [32.8 kB] 2024-04-13T07:16:07.242 INFO:teuthology.orchestra.run.smithi089.stdout:Get:37 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-cephfs amd64 18.2.2-1217-g7d36322c-1jammy [180 kB] 2024-04-13T07:16:07.248 INFO:teuthology.orchestra.run.smithi089.stdout:Get:38 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-ceph-common all 18.2.2-1217-g7d36322c-1jammy [63.7 kB] 2024-04-13T07:16:07.255 INFO:teuthology.orchestra.run.smithi089.stdout:Get:39 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-rbd amd64 18.2.2-1217-g7d36322c-1jammy [330 kB] 2024-04-13T07:16:07.274 INFO:teuthology.orchestra.run.smithi089.stdout:Get:40 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 librgw2 amd64 18.2.2-1217-g7d36322c-1jammy [5,891 kB] 2024-04-13T07:16:07.311 INFO:teuthology.orchestra.run.smithi203.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-04-13T07:16:07.339 INFO:teuthology.orchestra.run.smithi089.stdout:Get:41 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-pastedeploy-tpl all 2.1.1-1 [4,892 B] 2024-04-13T07:16:07.340 INFO:teuthology.orchestra.run.smithi089.stdout:Get:42 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastedeploy all 2.1.1-1 [26.6 kB] 2024-04-13T07:16:07.470 INFO:teuthology.orchestra.run.smithi203.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-04-13T07:16:07.499 INFO:teuthology.orchestra.run.smithi089.stdout:Get:43 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 python3-rgw amd64 18.2.2-1217-g7d36322c-1jammy [112 kB] 2024-04-13T07:16:07.502 INFO:teuthology.orchestra.run.smithi089.stdout:Get:44 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 libradosstriper1 amd64 18.2.2-1217-g7d36322c-1jammy [441 kB] 2024-04-13T07:16:07.508 INFO:teuthology.orchestra.run.smithi089.stdout:Get:45 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-webtest all 2.0.35-1 [28.5 kB] 2024-04-13T07:16:07.516 INFO:teuthology.orchestra.run.smithi089.stdout:Get:46 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-common amd64 18.2.2-1217-g7d36322c-1jammy [21.9 MB] 2024-04-13T07:16:07.575 INFO:teuthology.orchestra.run.smithi203.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-idna all 3.3-1 [49.3 kB] 2024-04-13T07:16:07.599 INFO:teuthology.orchestra.run.smithi203.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-04-13T07:16:07.628 INFO:teuthology.orchestra.run.smithi203.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-04-13T07:16:07.641 INFO:teuthology.orchestra.run.smithi203.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.1 [181 kB] 2024-04-13T07:16:07.682 INFO:teuthology.orchestra.run.smithi203.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-04-13T07:16:07.689 INFO:teuthology.orchestra.run.smithi203.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-04-13T07:16:07.704 INFO:teuthology.orchestra.run.smithi089.stdout:Get:47 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pecan all 1.3.3-4ubuntu2 [87.3 kB] 2024-04-13T07:16:07.719 INFO:teuthology.orchestra.run.smithi203.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-04-13T07:16:07.864 INFO:teuthology.orchestra.run.smithi089.stdout:Get:48 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-certifi all 2020.6.20-1 [150 kB] 2024-04-13T07:16:07.967 INFO:teuthology.orchestra.run.smithi089.stdout:Get:49 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-idna all 3.3-1 [49.3 kB] 2024-04-13T07:16:07.989 INFO:teuthology.orchestra.run.smithi089.stdout:Get:50 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-urllib3 all 1.26.5-1~exp1ubuntu0.1 [98.2 kB] 2024-04-13T07:16:07.991 INFO:teuthology.orchestra.run.smithi203.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-04-13T07:16:08.020 INFO:teuthology.orchestra.run.smithi089.stdout:Get:51 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-requests all 2.25.1+dfsg-2ubuntu0.1 [48.8 kB] 2024-04-13T07:16:08.032 INFO:teuthology.orchestra.run.smithi089.stdout:Get:52 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-werkzeug all 2.0.2+dfsg1-1ubuntu0.22.04.1 [181 kB] 2024-04-13T07:16:08.041 INFO:teuthology.orchestra.run.smithi203.stdout:Get:63 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 18.2.2-1217-g7d36322c-1jammy [14.1 kB] 2024-04-13T07:16:08.042 INFO:teuthology.orchestra.run.smithi203.stdout:Get:64 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 18.2.2-1217-g7d36322c-1jammy [831 kB] 2024-04-13T07:16:08.059 INFO:teuthology.orchestra.run.smithi203.stdout:Get:65 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 18.2.2-1217-g7d36322c-1jammy [2,154 kB] 2024-04-13T07:16:08.073 INFO:teuthology.orchestra.run.smithi089.stdout:Get:53 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-openssl all 21.0.0-1 [45.2 kB] 2024-04-13T07:16:08.081 INFO:teuthology.orchestra.run.smithi089.stdout:Get:54 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-asyncssh all 2.5.0-1 [185 kB] 2024-04-13T07:16:08.110 INFO:teuthology.orchestra.run.smithi089.stdout:Get:55 http://archive.ubuntu.com/ubuntu jammy/main amd64 python-babel-localedata all 2.8.0+dfsg.1-7 [4,982 kB] 2024-04-13T07:16:08.117 INFO:teuthology.orchestra.run.smithi203.stdout:Get:66 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 18.2.2-1217-g7d36322c-1jammy [238 kB] 2024-04-13T07:16:08.122 INFO:teuthology.orchestra.run.smithi203.stdout:Get:67 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 18.2.2-1217-g7d36322c-1jammy [144 kB] 2024-04-13T07:16:08.125 INFO:teuthology.orchestra.run.smithi203.stdout:Get:68 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 18.2.2-1217-g7d36322c-1jammy [2,239 kB] 2024-04-13T07:16:08.205 INFO:teuthology.orchestra.run.smithi203.stdout:Get:69 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.1 [108 kB] 2024-04-13T07:16:08.250 INFO:teuthology.orchestra.run.smithi203.stdout:Get:70 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 18.2.2-1217-g7d36322c-1jammy [8,625 kB] 2024-04-13T07:16:08.335 INFO:teuthology.orchestra.run.smithi089.stdout:Get:56 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-babel all 2.8.0+dfsg.1-7 [85.1 kB] 2024-04-13T07:16:08.500 INFO:teuthology.orchestra.run.smithi089.stdout:Get:57 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 python3-jinja2 all 3.0.3-1ubuntu0.1 [108 kB] 2024-04-13T07:16:08.564 INFO:teuthology.orchestra.run.smithi203.stdout:Get:71 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-04-13T07:16:08.581 INFO:teuthology.orchestra.run.smithi203.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-04-13T07:16:08.593 INFO:teuthology.orchestra.run.smithi203.stdout:Get:73 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-04-13T07:16:08.646 INFO:teuthology.orchestra.run.smithi203.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-04-13T07:16:08.650 INFO:teuthology.orchestra.run.smithi203.stdout:Get:75 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-04-13T07:16:08.659 INFO:teuthology.orchestra.run.smithi203.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-04-13T07:16:08.739 INFO:teuthology.orchestra.run.smithi203.stdout:Get:77 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 18.2.2-1217-g7d36322c-1jammy [14.3 kB] 2024-04-13T07:16:08.739 INFO:teuthology.orchestra.run.smithi203.stdout:Get:78 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 18.2.2-1217-g7d36322c-1jammy [42.6 MB] 2024-04-13T07:16:08.746 INFO:teuthology.orchestra.run.smithi203.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-04-13T07:16:08.749 INFO:teuthology.orchestra.run.smithi203.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-04-13T07:16:08.769 INFO:teuthology.orchestra.run.smithi203.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-04-13T07:16:08.858 INFO:teuthology.orchestra.run.smithi089.stdout:Get:58 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-bcrypt amd64 3.2.0-1build1 [32.7 kB] 2024-04-13T07:16:08.875 INFO:teuthology.orchestra.run.smithi089.stdout:Get:59 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pluggy all 0.13.0-7.1 [19.0 kB] 2024-04-13T07:16:08.887 INFO:teuthology.orchestra.run.smithi089.stdout:Get:60 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-py all 1.10.0-1 [71.9 kB] 2024-04-13T07:16:08.946 INFO:teuthology.orchestra.run.smithi089.stdout:Get:61 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-iniconfig all 1.1.1-2 [6,024 B] 2024-04-13T07:16:08.947 INFO:teuthology.orchestra.run.smithi089.stdout:Get:62 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-toml all 0.10.2-1 [16.5 kB] 2024-04-13T07:16:08.963 INFO:teuthology.orchestra.run.smithi203.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-04-13T07:16:09.061 INFO:teuthology.orchestra.run.smithi089.stdout:Get:63 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-pytest all 6.2.5-1ubuntu2 [214 kB] 2024-04-13T07:16:09.062 INFO:teuthology.orchestra.run.smithi089.stdout:Get:64 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-base amd64 18.2.2-1217-g7d36322c-1jammy [5,002 kB] 2024-04-13T07:16:09.070 INFO:teuthology.orchestra.run.smithi089.stdout:Get:65 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-repoze.lru all 0.7-2 [12.1 kB] 2024-04-13T07:16:09.071 INFO:teuthology.orchestra.run.smithi089.stdout:Get:66 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-routes all 2.5.1-1ubuntu1 [89.0 kB] 2024-04-13T07:16:09.076 INFO:teuthology.orchestra.run.smithi089.stdout:Get:67 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn-lib amd64 0.23.2-5ubuntu6 [2,058 kB] 2024-04-13T07:16:09.135 INFO:teuthology.orchestra.run.smithi203.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-04-13T07:16:09.255 INFO:teuthology.orchestra.run.smithi089.stdout:Get:68 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-joblib all 0.17.0-4ubuntu1 [204 kB] 2024-04-13T07:16:09.317 INFO:teuthology.orchestra.run.smithi203.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-04-13T07:16:09.331 INFO:teuthology.orchestra.run.smithi089.stdout:Get:69 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-modules-core all 18.2.2-1217-g7d36322c-1jammy [241 kB] 2024-04-13T07:16:09.361 INFO:teuthology.orchestra.run.smithi089.stdout:Get:70 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 libsqlite3-mod-ceph amd64 18.2.2-1217-g7d36322c-1jammy [130 kB] 2024-04-13T07:16:09.366 INFO:teuthology.orchestra.run.smithi089.stdout:Get:71 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr amd64 18.2.2-1217-g7d36322c-1jammy [1,490 kB] 2024-04-13T07:16:09.426 INFO:teuthology.orchestra.run.smithi089.stdout:Get:72 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-threadpoolctl all 3.1.0-1 [21.3 kB] 2024-04-13T07:16:09.480 INFO:teuthology.orchestra.run.smithi089.stdout:Get:73 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mon amd64 18.2.2-1217-g7d36322c-1jammy [6,079 kB] 2024-04-13T07:16:09.607 INFO:teuthology.orchestra.run.smithi089.stdout:Get:74 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-sklearn all 0.23.2-5ubuntu6 [1,829 kB] 2024-04-13T07:16:09.831 INFO:teuthology.orchestra.run.smithi203.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-04-13T07:16:09.831 INFO:teuthology.orchestra.run.smithi203.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-04-13T07:16:09.834 INFO:teuthology.orchestra.run.smithi203.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-04-13T07:16:09.836 INFO:teuthology.orchestra.run.smithi203.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-04-13T07:16:09.837 INFO:teuthology.orchestra.run.smithi203.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-04-13T07:16:09.840 INFO:teuthology.orchestra.run.smithi203.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-04-13T07:16:09.841 INFO:teuthology.orchestra.run.smithi203.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-04-13T07:16:09.843 INFO:teuthology.orchestra.run.smithi203.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-04-13T07:16:09.861 INFO:teuthology.orchestra.run.smithi203.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-04-13T07:16:09.952 INFO:teuthology.orchestra.run.smithi089.stdout:Get:75 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-osd amd64 18.2.2-1217-g7d36322c-1jammy [22.3 MB] 2024-04-13T07:16:10.034 INFO:teuthology.orchestra.run.smithi203.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-04-13T07:16:10.122 INFO:teuthology.orchestra.run.smithi089.stdout:Get:76 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-cachetools all 5.0.0-1 [9,722 B] 2024-04-13T07:16:10.123 INFO:teuthology.orchestra.run.smithi089.stdout:Get:77 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1 all 0.4.8-1 [50.9 kB] 2024-04-13T07:16:10.125 INFO:teuthology.orchestra.run.smithi089.stdout:Get:78 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyasn1-modules all 0.2.1-1 [33.0 kB] 2024-04-13T07:16:10.127 INFO:teuthology.orchestra.run.smithi089.stdout:Get:79 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-rsa all 4.8-1 [28.4 kB] 2024-04-13T07:16:10.129 INFO:teuthology.orchestra.run.smithi089.stdout:Get:80 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-google-auth all 1.5.1-3 [35.7 kB] 2024-04-13T07:16:10.131 INFO:teuthology.orchestra.run.smithi089.stdout:Get:81 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-requests-oauthlib all 1.3.0+ds-0.1 [18.7 kB] 2024-04-13T07:16:10.132 INFO:teuthology.orchestra.run.smithi089.stdout:Get:82 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-websocket all 1.2.3-1 [34.7 kB] 2024-04-13T07:16:10.134 INFO:teuthology.orchestra.run.smithi089.stdout:Get:83 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python3-kubernetes all 12.0.1-1ubuntu1 [353 kB] 2024-04-13T07:16:10.151 INFO:teuthology.orchestra.run.smithi089.stdout:Get:84 http://archive.ubuntu.com/ubuntu jammy/main amd64 libonig5 amd64 6.9.7.1-2build1 [172 kB] 2024-04-13T07:16:10.323 INFO:teuthology.orchestra.run.smithi089.stdout:Get:85 http://archive.ubuntu.com/ubuntu jammy/main amd64 libjq1 amd64 1.6-2.1ubuntu3 [133 kB] 2024-04-13T07:16:10.406 INFO:teuthology.orchestra.run.smithi203.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-04-13T07:16:10.431 INFO:teuthology.orchestra.run.smithi203.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-04-13T07:16:10.582 INFO:teuthology.orchestra.run.smithi203.stdout:Get:97 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-04-13T07:16:10.633 INFO:teuthology.orchestra.run.smithi203.stdout:Get:98 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB] 2024-04-13T07:16:10.694 INFO:teuthology.orchestra.run.smithi203.stdout:Get:99 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-04-13T07:16:10.704 INFO:teuthology.orchestra.run.smithi089.stdout:Get:86 http://archive.ubuntu.com/ubuntu jammy/main amd64 jq amd64 1.6-2.1ubuntu3 [52.5 kB] 2024-04-13T07:16:10.819 INFO:teuthology.orchestra.run.smithi089.stdout:Get:87 http://archive.ubuntu.com/ubuntu jammy/main amd64 socat amd64 1.7.4.1-3ubuntu4 [349 kB] 2024-04-13T07:16:10.820 INFO:teuthology.orchestra.run.smithi203.stdout:Get:100 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-04-13T07:16:10.820 INFO:teuthology.orchestra.run.smithi203.stdout:Get:101 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-04-13T07:16:10.820 INFO:teuthology.orchestra.run.smithi203.stdout:Get:102 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-04-13T07:16:10.820 INFO:teuthology.orchestra.run.smithi203.stdout:Get:103 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-04-13T07:16:10.820 INFO:teuthology.orchestra.run.smithi203.stdout:Get:104 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-04-13T07:16:10.879 INFO:teuthology.orchestra.run.smithi089.stdout:Get:88 http://archive.ubuntu.com/ubuntu jammy/universe amd64 xmlstarlet amd64 1.6.1-2.1 [265 kB] 2024-04-13T07:16:10.929 INFO:teuthology.orchestra.run.smithi089.stdout:Get:89 http://archive.ubuntu.com/ubuntu jammy-updates/main amd64 nvme-cli amd64 1.16-3ubuntu0.1 [474 kB] 2024-04-13T07:16:10.947 INFO:teuthology.orchestra.run.smithi203.stdout:Get:105 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-04-13T07:16:10.991 INFO:teuthology.orchestra.run.smithi089.stdout:Get:90 http://archive.ubuntu.com/ubuntu jammy/universe amd64 python-asyncssh-doc all 2.5.0-1 [318 kB] 2024-04-13T07:16:11.017 INFO:teuthology.orchestra.run.smithi089.stdout:Get:91 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pastescript all 2.0.2-4 [54.6 kB] 2024-04-13T07:16:11.021 INFO:teuthology.orchestra.run.smithi089.stdout:Get:92 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-psutil amd64 5.9.0-1build1 [158 kB] 2024-04-13T07:16:11.033 INFO:teuthology.orchestra.run.smithi089.stdout:Get:93 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pygments all 2.11.2+dfsg-2 [750 kB] 2024-04-13T07:16:11.076 INFO:teuthology.orchestra.run.smithi089.stdout:Get:94 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-pyinotify all 0.9.6-1.3 [24.8 kB] 2024-04-13T07:16:11.077 INFO:teuthology.orchestra.run.smithi089.stdout:Get:95 http://archive.ubuntu.com/ubuntu jammy/main amd64 python3-simplejson amd64 3.17.6-1build1 [54.7 kB] 2024-04-13T07:16:11.219 INFO:teuthology.orchestra.run.smithi203.stdout:Get:106 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 18.2.2-1217-g7d36322c-1jammy [121 kB] 2024-04-13T07:16:11.222 INFO:teuthology.orchestra.run.smithi203.stdout:Get:107 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 18.2.2-1217-g7d36322c-1jammy [38.3 kB] 2024-04-13T07:16:11.223 INFO:teuthology.orchestra.run.smithi203.stdout:Get:108 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 18.2.2-1217-g7d36322c-1jammy [11.6 MB] 2024-04-13T07:16:11.243 INFO:teuthology.orchestra.run.smithi089.stdout:Get:96 http://archive.ubuntu.com/ubuntu jammy/universe amd64 qttranslations5-l10n all 5.15.3-1 [1,983 kB] 2024-04-13T07:16:11.659 INFO:teuthology.orchestra.run.smithi203.stdout:Get:109 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 18.2.2-1217-g7d36322c-1jammy [93.1 kB] 2024-04-13T07:16:11.865 INFO:teuthology.orchestra.run.smithi089.stdout:Get:97 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph amd64 18.2.2-1217-g7d36322c-1jammy [14.1 kB] 2024-04-13T07:16:11.866 INFO:teuthology.orchestra.run.smithi089.stdout:Get:98 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-fuse amd64 18.2.2-1217-g7d36322c-1jammy [831 kB] 2024-04-13T07:16:11.882 INFO:teuthology.orchestra.run.smithi089.stdout:Get:99 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mds amd64 18.2.2-1217-g7d36322c-1jammy [2,154 kB] 2024-04-13T07:16:11.919 INFO:teuthology.orchestra.run.smithi089.stdout:Get:100 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 cephadm amd64 18.2.2-1217-g7d36322c-1jammy [238 kB] 2024-04-13T07:16:11.927 INFO:teuthology.orchestra.run.smithi089.stdout:Get:101 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-cephadm all 18.2.2-1217-g7d36322c-1jammy [144 kB] 2024-04-13T07:16:11.931 INFO:teuthology.orchestra.run.smithi089.stdout:Get:102 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-dashboard all 18.2.2-1217-g7d36322c-1jammy [2,239 kB] 2024-04-13T07:16:12.005 INFO:teuthology.orchestra.run.smithi089.stdout:Get:103 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-diskprediction-local all 18.2.2-1217-g7d36322c-1jammy [8,625 kB] 2024-04-13T07:16:12.214 INFO:teuthology.orchestra.run.smithi203.stdout:Fetched 162 MB in 7s (22.7 MB/s) 2024-04-13T07:16:12.289 INFO:teuthology.orchestra.run.smithi089.stdout:Get:104 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-mgr-k8sevents all 18.2.2-1217-g7d36322c-1jammy [14.3 kB] 2024-04-13T07:16:12.290 INFO:teuthology.orchestra.run.smithi089.stdout:Get:105 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-test amd64 18.2.2-1217-g7d36322c-1jammy [42.6 MB] 2024-04-13T07:16:12.352 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libfmt8:amd64. 2024-04-13T07:16:12.449 INFO:teuthology.orchestra.run.smithi203.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165995 files and directories currently installed.) 2024-04-13T07:16:12.454 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-04-13T07:16:12.542 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-04-13T07:16:12.853 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-04-13T07:16:12.869 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-04-13T07:16:12.903 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-04-13T07:16:13.224 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-04-13T07:16:13.250 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../002-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-04-13T07:16:13.290 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-04-13T07:16:13.635 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-04-13T07:16:13.661 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../003-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-04-13T07:16:13.701 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-04-13T07:16:13.868 INFO:teuthology.orchestra.run.smithi089.stdout:Get:106 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 ceph-volume all 18.2.2-1217-g7d36322c-1jammy [121 kB] 2024-04-13T07:16:13.871 INFO:teuthology.orchestra.run.smithi089.stdout:Get:107 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 libcephfs-dev amd64 18.2.2-1217-g7d36322c-1jammy [38.3 kB] 2024-04-13T07:16:13.872 INFO:teuthology.orchestra.run.smithi089.stdout:Get:108 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 radosgw amd64 18.2.2-1217-g7d36322c-1jammy [11.6 MB] 2024-04-13T07:16:14.146 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-04-13T07:16:14.168 INFO:teuthology.orchestra.run.smithi089.stdout:Get:109 https://1.chacra.ceph.com/r/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/flavors/default jammy/main amd64 rbd-fuse amd64 18.2.2-1217-g7d36322c-1jammy [93.1 kB] 2024-04-13T07:16:14.173 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../004-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-04-13T07:16:14.221 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:16:14.591 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-04-13T07:16:14.617 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../005-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-04-13T07:16:14.658 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:16:14.722 INFO:teuthology.orchestra.run.smithi089.stdout:Fetched 162 MB in 9s (17.6 MB/s) 2024-04-13T07:16:14.856 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libfmt8:amd64. 2024-04-13T07:16:14.955 INFO:teuthology.orchestra.run.smithi089.stdout:(Reading database ... (Reading database ... 5% (Reading database ... 10% (Reading database ... 15% (Reading database ... 20% (Reading database ... 25% (Reading database ... 30% (Reading database ... 35% (Reading database ... 40% (Reading database ... 45% (Reading database ... 50% (Reading database ... 55% (Reading database ... 60% (Reading database ... 65% (Reading database ... 70% (Reading database ... 75% (Reading database ... 80% (Reading database ... 85% (Reading database ... 90% (Reading database ... 95% (Reading database ... 100% (Reading database ... 165995 files and directories currently installed.) 2024-04-13T07:16:14.960 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../000-libfmt8_8.1.1+ds1-2_amd64.deb ... 2024-04-13T07:16:15.019 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-04-13T07:16:15.046 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../006-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-04-13T07:16:15.054 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libfmt8:amd64 (8.1.1+ds1-2) ... 2024-04-13T07:16:15.086 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:16:15.357 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package liblttng-ust1:amd64. 2024-04-13T07:16:15.372 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../001-liblttng-ust1_2.13.1-1ubuntu1_amd64.deb ... 2024-04-13T07:16:15.407 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-04-13T07:16:15.456 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-04-13T07:16:15.482 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../007-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-04-13T07:16:15.522 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-04-13T07:16:15.719 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libdouble-conversion3:amd64. 2024-04-13T07:16:15.746 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../002-libdouble-conversion3_3.1.7-4_amd64.deb ... 2024-04-13T07:16:15.785 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libdouble-conversion3:amd64 (3.1.7-4) ... 2024-04-13T07:16:15.850 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package librados2. 2024-04-13T07:16:15.877 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../008-librados2_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:15.916 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking librados2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:16.122 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libpcre2-16-0:amd64. 2024-04-13T07:16:16.148 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../003-libpcre2-16-0_10.39-3ubuntu0.1_amd64.deb ... 2024-04-13T07:16:16.188 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-04-13T07:16:16.336 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package librbd1. 2024-04-13T07:16:16.362 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../009-librbd1_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:16.394 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking librbd1 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:16.659 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libqt5core5a:amd64. 2024-04-13T07:16:16.685 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../004-libqt5core5a_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-04-13T07:16:16.733 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:16:16.864 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libcephfs2. 2024-04-13T07:16:16.890 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../010-libcephfs2_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:16.930 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libcephfs2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:17.145 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libqt5dbus5:amd64. 2024-04-13T07:16:17.172 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../005-libqt5dbus5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-04-13T07:16:17.212 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:16:17.216 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-rados. 2024-04-13T07:16:17.243 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../011-python3-rados_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:17.282 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-rados (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:17.669 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-04-13T07:16:17.695 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../012-python3-ceph-argparse_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:17.735 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-ceph-argparse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:17.756 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libqt5network5:amd64. 2024-04-13T07:16:17.783 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../006-libqt5network5_5.15.3+dfsg-2ubuntu0.2_amd64.deb ... 2024-04-13T07:16:17.823 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:16:17.962 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-cephfs. 2024-04-13T07:16:17.978 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../013-python3-cephfs_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:18.012 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-cephfs (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:18.151 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libthrift-0.16.0:amd64. 2024-04-13T07:16:18.178 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../007-libthrift-0.16.0_0.16.0-2_amd64.deb ... 2024-04-13T07:16:18.217 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-04-13T07:16:18.332 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-ceph-common. 2024-04-13T07:16:18.358 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../014-python3-ceph-common_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:18.398 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-ceph-common (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:18.496 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librados2. 2024-04-13T07:16:18.522 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../008-librados2_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:18.562 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librados2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:18.710 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-wcwidth. 2024-04-13T07:16:18.737 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-04-13T07:16:18.777 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-04-13T07:16:19.007 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librbd1. 2024-04-13T07:16:19.033 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../009-librbd1_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:19.073 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librbd1 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:19.104 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-prettytable. 2024-04-13T07:16:19.130 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2024-04-13T07:16:19.171 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-04-13T07:16:19.432 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-rbd. 2024-04-13T07:16:19.458 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../017-python3-rbd_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:19.490 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-rbd (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:19.518 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libcephfs2. 2024-04-13T07:16:19.544 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../010-libcephfs2_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:19.601 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libcephfs2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:19.826 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-04-13T07:16:19.853 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-04-13T07:16:19.870 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rados. 2024-04-13T07:16:19.886 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../011-python3-rados_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:19.893 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-04-13T07:16:19.920 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rados (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:20.223 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-ceph-argparse. 2024-04-13T07:16:20.229 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package librgw2. 2024-04-13T07:16:20.249 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../012-python3-ceph-argparse_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:20.255 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../019-librgw2_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:20.290 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-ceph-argparse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:20.295 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking librgw2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:20.542 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cephfs. 2024-04-13T07:16:20.557 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../013-python3-cephfs_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:20.609 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cephfs (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:20.806 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-rgw. 2024-04-13T07:16:20.823 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../020-python3-rgw_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:20.864 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-rgw (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:20.912 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-ceph-common. 2024-04-13T07:16:20.938 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../014-python3-ceph-common_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:20.978 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-ceph-common (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:21.167 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package liboath0:amd64. 2024-04-13T07:16:21.194 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-04-13T07:16:21.225 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-04-13T07:16:21.264 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-wcwidth. 2024-04-13T07:16:21.279 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../015-python3-wcwidth_0.2.5+dfsg1-1_all.deb ... 2024-04-13T07:16:21.314 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-wcwidth (0.2.5+dfsg1-1) ... 2024-04-13T07:16:21.578 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libradosstriper1. 2024-04-13T07:16:21.600 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-prettytable. 2024-04-13T07:16:21.604 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../022-libradosstriper1_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:21.627 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../016-python3-prettytable_2.5.0-2_all.deb ... 2024-04-13T07:16:21.645 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libradosstriper1 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:21.667 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-prettytable (2.5.0-2) ... 2024-04-13T07:16:21.912 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rbd. 2024-04-13T07:16:21.938 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../017-python3-rbd_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:21.978 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rbd (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:22.090 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-common. 2024-04-13T07:16:22.117 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../023-ceph-common_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:22.156 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-common (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:22.340 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librdkafka1:amd64. 2024-04-13T07:16:22.366 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../018-librdkafka1_1.8.0-1build1_amd64.deb ... 2024-04-13T07:16:22.406 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librdkafka1:amd64 (1.8.0-1build1) ... 2024-04-13T07:16:22.709 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package librgw2. 2024-04-13T07:16:22.735 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../019-librgw2_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:22.776 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking librgw2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:23.453 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rgw. 2024-04-13T07:16:23.480 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../020-python3-rgw_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:23.520 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rgw (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:23.604 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-base. 2024-04-13T07:16:23.630 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../024-ceph-base_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:23.679 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-base (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:23.856 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package liboath0:amd64. 2024-04-13T07:16:23.883 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../021-liboath0_2.6.7-3build1_amd64.deb ... 2024-04-13T07:16:23.923 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking liboath0:amd64 (2.6.7-3build1) ... 2024-04-13T07:16:24.234 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libradosstriper1. 2024-04-13T07:16:24.252 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-04-13T07:16:24.261 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../022-libradosstriper1_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:24.279 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../025-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-04-13T07:16:24.300 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libradosstriper1 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:24.318 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-04-13T07:16:24.580 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-cheroot. 2024-04-13T07:16:24.606 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../026-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-04-13T07:16:24.638 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-04-13T07:16:24.721 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-common. 2024-04-13T07:16:24.747 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../023-ceph-common_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:24.787 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-common (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:24.933 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-04-13T07:16:24.949 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../027-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-04-13T07:16:24.991 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-04-13T07:16:25.278 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-jaraco.text. 2024-04-13T07:16:25.305 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../028-python3-jaraco.text_3.6.0-2_all.deb ... 2024-04-13T07:16:25.344 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-04-13T07:16:25.647 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-04-13T07:16:25.674 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../029-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-04-13T07:16:25.714 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-04-13T07:16:26.009 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-tempora. 2024-04-13T07:16:26.035 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../030-python3-tempora_4.1.2-1_all.deb ... 2024-04-13T07:16:26.075 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-04-13T07:16:26.076 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-base. 2024-04-13T07:16:26.103 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../024-ceph-base_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:26.153 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-base (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:26.345 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-portend. 2024-04-13T07:16:26.371 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../031-python3-portend_3.0.0-1_all.deb ... 2024-04-13T07:16:26.412 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-04-13T07:16:26.715 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-04-13T07:16:26.739 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jaraco.functools. 2024-04-13T07:16:26.741 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../032-python3-zc.lockfile_2.0-1_all.deb ... 2024-04-13T07:16:26.766 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../025-python3-jaraco.functools_3.4.0-2_all.deb ... 2024-04-13T07:16:26.781 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-04-13T07:16:26.806 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jaraco.functools (3.4.0-2) ... 2024-04-13T07:16:27.066 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cheroot. 2024-04-13T07:16:27.082 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../026-python3-cheroot_8.5.2+ds1-1ubuntu3.1_all.deb ... 2024-04-13T07:16:27.101 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-cherrypy3. 2024-04-13T07:16:27.116 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-04-13T07:16:27.127 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../033-python3-cherrypy3_18.6.1-4_all.deb ... 2024-04-13T07:16:27.168 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-04-13T07:16:27.428 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jaraco.classes. 2024-04-13T07:16:27.455 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../027-python3-jaraco.classes_3.2.1-3_all.deb ... 2024-04-13T07:16:27.495 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jaraco.classes (3.2.1-3) ... 2024-04-13T07:16:27.496 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-natsort. 2024-04-13T07:16:27.523 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../034-python3-natsort_8.0.2-1_all.deb ... 2024-04-13T07:16:27.563 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-04-13T07:16:27.772 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jaraco.text. 2024-04-13T07:16:27.788 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../028-python3-jaraco.text_3.6.0-2_all.deb ... 2024-04-13T07:16:27.822 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jaraco.text (3.6.0-2) ... 2024-04-13T07:16:28.025 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-logutils. 2024-04-13T07:16:28.051 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../035-python3-logutils_0.3.3-8_all.deb ... 2024-04-13T07:16:28.083 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-04-13T07:16:28.108 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jaraco.collections. 2024-04-13T07:16:28.135 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../029-python3-jaraco.collections_3.4.0-2_all.deb ... 2024-04-13T07:16:28.175 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jaraco.collections (3.4.0-2) ... 2024-04-13T07:16:28.353 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-markupsafe. 2024-04-13T07:16:28.379 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../036-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-04-13T07:16:28.419 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-04-13T07:16:28.478 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-tempora. 2024-04-13T07:16:28.508 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../030-python3-tempora_4.1.2-1_all.deb ... 2024-04-13T07:16:28.711 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-tempora (4.1.2-1) ... 2024-04-13T07:16:28.722 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-mako. 2024-04-13T07:16:28.749 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../037-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-04-13T07:16:28.789 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-04-13T07:16:29.039 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-portend. 2024-04-13T07:16:29.066 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../031-python3-portend_3.0.0-1_all.deb ... 2024-04-13T07:16:29.101 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-simplegeneric. 2024-04-13T07:16:29.106 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-portend (3.0.0-1) ... 2024-04-13T07:16:29.120 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../038-python3-simplegeneric_0.8.1-3_all.deb ... 2024-04-13T07:16:29.159 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-04-13T07:16:29.409 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-zc.lockfile. 2024-04-13T07:16:29.435 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../032-python3-zc.lockfile_2.0-1_all.deb ... 2024-04-13T07:16:29.462 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-singledispatch. 2024-04-13T07:16:29.475 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-zc.lockfile (2.0-1) ... 2024-04-13T07:16:29.489 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../039-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-04-13T07:16:29.545 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-04-13T07:16:29.770 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cherrypy3. 2024-04-13T07:16:29.796 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../033-python3-cherrypy3_18.6.1-4_all.deb ... 2024-04-13T07:16:29.828 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cherrypy3 (18.6.1-4) ... 2024-04-13T07:16:29.832 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-webob. 2024-04-13T07:16:29.858 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../040-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-04-13T07:16:29.898 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-04-13T07:16:30.148 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-natsort. 2024-04-13T07:16:30.164 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../034-python3-natsort_8.0.2-1_all.deb ... 2024-04-13T07:16:30.198 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-natsort (8.0.2-1) ... 2024-04-13T07:16:30.218 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-waitress. 2024-04-13T07:16:30.245 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../041-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-04-13T07:16:30.280 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-04-13T07:16:30.493 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-logutils. 2024-04-13T07:16:30.520 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../035-python3-logutils_0.3.3-8_all.deb ... 2024-04-13T07:16:30.555 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-tempita. 2024-04-13T07:16:30.559 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-logutils (0.3.3-8) ... 2024-04-13T07:16:30.581 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../042-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-04-13T07:16:30.621 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-04-13T07:16:30.871 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-markupsafe. 2024-04-13T07:16:30.898 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../036-python3-markupsafe_2.0.1-2build1_amd64.deb ... 2024-04-13T07:16:30.924 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-paste. 2024-04-13T07:16:30.937 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-markupsafe (2.0.1-2build1) ... 2024-04-13T07:16:30.948 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../043-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-04-13T07:16:30.999 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-04-13T07:16:31.240 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-mako. 2024-04-13T07:16:31.267 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../037-python3-mako_1.1.3+ds1-2ubuntu0.1_all.deb ... 2024-04-13T07:16:31.303 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-04-13T07:16:31.307 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-04-13T07:16:31.318 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../044-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-04-13T07:16:31.361 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-04-13T07:16:31.627 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-simplegeneric. 2024-04-13T07:16:31.654 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../038-python3-simplegeneric_0.8.1-3_all.deb ... 2024-04-13T07:16:31.664 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pastedeploy. 2024-04-13T07:16:31.690 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../045-python3-pastedeploy_2.1.1-1_all.deb ... 2024-04-13T07:16:31.693 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-simplegeneric (0.8.1-3) ... 2024-04-13T07:16:31.730 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-04-13T07:16:32.005 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-singledispatch. 2024-04-13T07:16:32.074 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-webtest. 2024-04-13T07:16:32.074 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../046-python3-webtest_2.0.35-1_all.deb ... 2024-04-13T07:16:32.074 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../039-python3-singledispatch_3.4.0.3-3_all.deb ... 2024-04-13T07:16:32.075 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-singledispatch (3.4.0.3-3) ... 2024-04-13T07:16:32.109 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-04-13T07:16:32.374 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-webob. 2024-04-13T07:16:32.401 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../040-python3-webob_1%3a1.8.6-1.1_all.deb ... 2024-04-13T07:16:32.404 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pecan. 2024-04-13T07:16:32.430 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../047-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-04-13T07:16:32.440 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-webob (1:1.8.6-1.1) ... 2024-04-13T07:16:32.462 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-04-13T07:16:32.760 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-waitress. 2024-04-13T07:16:32.783 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../041-python3-waitress_1.4.4-1.1ubuntu1_all.deb ... 2024-04-13T07:16:32.821 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-waitress (1.4.4-1.1ubuntu1) ... 2024-04-13T07:16:32.826 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-certifi. 2024-04-13T07:16:32.852 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../048-python3-certifi_2020.6.20-1_all.deb ... 2024-04-13T07:16:32.884 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-04-13T07:16:33.114 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-tempita. 2024-04-13T07:16:33.140 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../042-python3-tempita_0.5.2-6ubuntu1_all.deb ... 2024-04-13T07:16:33.180 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-tempita (0.5.2-6ubuntu1) ... 2024-04-13T07:16:33.207 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-idna. 2024-04-13T07:16:33.232 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../049-python3-idna_3.3-1_all.deb ... 2024-04-13T07:16:33.270 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-idna (3.3-1) ... 2024-04-13T07:16:33.466 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-paste. 2024-04-13T07:16:33.493 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../043-python3-paste_3.5.0+dfsg1-1_all.deb ... 2024-04-13T07:16:33.533 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-paste (3.5.0+dfsg1-1) ... 2024-04-13T07:16:33.590 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-urllib3. 2024-04-13T07:16:33.616 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../050-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-04-13T07:16:33.656 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-04-13T07:16:33.844 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python-pastedeploy-tpl. 2024-04-13T07:16:33.859 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../044-python-pastedeploy-tpl_2.1.1-1_all.deb ... 2024-04-13T07:16:33.893 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python-pastedeploy-tpl (2.1.1-1) ... 2024-04-13T07:16:33.985 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-requests. 2024-04-13T07:16:34.011 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../051-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-04-13T07:16:34.051 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-04-13T07:16:34.189 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pastedeploy. 2024-04-13T07:16:34.215 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../045-python3-pastedeploy_2.1.1-1_all.deb ... 2024-04-13T07:16:34.255 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pastedeploy (2.1.1-1) ... 2024-04-13T07:16:34.371 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-werkzeug. 2024-04-13T07:16:34.398 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../052-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.1_all.deb ... 2024-04-13T07:16:34.438 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-04-13T07:16:34.541 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-webtest. 2024-04-13T07:16:34.568 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../046-python3-webtest_2.0.35-1_all.deb ... 2024-04-13T07:16:34.608 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-webtest (2.0.35-1) ... 2024-04-13T07:16:34.741 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-04-13T07:16:34.756 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../053-ceph-mgr-modules-core_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:34.799 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-mgr-modules-core (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:34.902 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pecan. 2024-04-13T07:16:34.929 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../047-python3-pecan_1.3.3-4ubuntu2_all.deb ... 2024-04-13T07:16:34.969 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pecan (1.3.3-4ubuntu2) ... 2024-04-13T07:16:35.172 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-04-13T07:16:35.198 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../054-libsqlite3-mod-ceph_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:35.238 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libsqlite3-mod-ceph (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:35.332 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-certifi. 2024-04-13T07:16:35.359 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../048-python3-certifi_2020.6.20-1_all.deb ... 2024-04-13T07:16:35.398 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-certifi (2020.6.20-1) ... 2024-04-13T07:16:35.558 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-openssl. 2024-04-13T07:16:35.584 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../055-python3-openssl_21.0.0-1_all.deb ... 2024-04-13T07:16:35.624 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-04-13T07:16:35.676 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-idna. 2024-04-13T07:16:35.703 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../049-python3-idna_3.3-1_all.deb ... 2024-04-13T07:16:35.734 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-idna (3.3-1) ... 2024-04-13T07:16:35.978 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-mgr. 2024-04-13T07:16:36.004 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../056-ceph-mgr_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:36.044 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-mgr (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:36.056 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-urllib3. 2024-04-13T07:16:36.083 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../050-python3-urllib3_1.26.5-1~exp1ubuntu0.1_all.deb ... 2024-04-13T07:16:36.123 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-04-13T07:16:36.398 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-mon. 2024-04-13T07:16:36.425 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../057-ceph-mon_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:36.443 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-requests. 2024-04-13T07:16:36.455 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-mon (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:36.470 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../051-python3-requests_2.25.1+dfsg-2ubuntu0.1_all.deb ... 2024-04-13T07:16:36.509 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-04-13T07:16:36.812 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-werkzeug. 2024-04-13T07:16:36.839 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../052-python3-werkzeug_2.0.2+dfsg1-1ubuntu0.22.04.1_all.deb ... 2024-04-13T07:16:36.879 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-04-13T07:16:37.069 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-osd. 2024-04-13T07:16:37.095 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../058-ceph-osd_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:37.134 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-osd (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:37.174 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-modules-core. 2024-04-13T07:16:37.200 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../053-ceph-mgr-modules-core_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:37.240 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-modules-core (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:37.628 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libsqlite3-mod-ceph. 2024-04-13T07:16:37.656 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../054-libsqlite3-mod-ceph_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:37.695 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libsqlite3-mod-ceph (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:38.915 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-openssl. 2024-04-13T07:16:38.942 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../055-python3-openssl_21.0.0-1_all.deb ... 2024-04-13T07:16:38.981 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-openssl (21.0.0-1) ... 2024-04-13T07:16:39.088 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph. 2024-04-13T07:16:39.104 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../059-ceph_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:39.138 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:39.284 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr. 2024-04-13T07:16:39.311 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../056-ceph-mgr_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:39.342 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:39.432 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-fuse. 2024-04-13T07:16:39.459 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../060-ceph-fuse_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:39.499 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-fuse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:39.704 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mon. 2024-04-13T07:16:39.734 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../057-ceph-mon_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:39.771 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mon (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:39.895 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-mds. 2024-04-13T07:16:39.913 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../061-ceph-mds_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:39.952 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-mds (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:40.340 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package cephadm. 2024-04-13T07:16:40.366 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../062-cephadm_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:40.391 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-osd. 2024-04-13T07:16:40.406 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking cephadm (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:40.418 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../058-ceph-osd_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:40.457 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-osd (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:40.685 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-asyncssh. 2024-04-13T07:16:40.712 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../063-python3-asyncssh_2.5.0-1_all.deb ... 2024-04-13T07:16:40.751 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-04-13T07:16:41.079 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python-babel-localedata. 2024-04-13T07:16:41.096 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-04-13T07:16:41.128 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-04-13T07:16:41.494 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph. 2024-04-13T07:16:41.510 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../059-ceph_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:41.544 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:41.847 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-fuse. 2024-04-13T07:16:41.874 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../060-ceph-fuse_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:41.914 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-fuse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:42.143 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-babel. 2024-04-13T07:16:42.169 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-04-13T07:16:42.201 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-04-13T07:16:42.292 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mds. 2024-04-13T07:16:42.319 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../061-ceph-mds_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:42.358 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mds (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:42.504 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-jinja2. 2024-04-13T07:16:42.531 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.1_all.deb ... 2024-04-13T07:16:42.571 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-04-13T07:16:42.753 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package cephadm. 2024-04-13T07:16:42.780 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../062-cephadm_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:42.819 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking cephadm (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:42.916 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-04-13T07:16:42.943 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:42.982 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-mgr-cephadm (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:43.098 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-asyncssh. 2024-04-13T07:16:43.125 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../063-python3-asyncssh_2.5.0-1_all.deb ... 2024-04-13T07:16:43.164 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-asyncssh (2.5.0-1) ... 2024-04-13T07:16:43.312 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-bcrypt. 2024-04-13T07:16:43.338 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../068-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-04-13T07:16:43.378 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-04-13T07:16:43.459 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python-babel-localedata. 2024-04-13T07:16:43.486 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../064-python-babel-localedata_2.8.0+dfsg.1-7_all.deb ... 2024-04-13T07:16:43.517 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-04-13T07:16:43.681 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pluggy. 2024-04-13T07:16:43.708 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../069-python3-pluggy_0.13.0-7.1_all.deb ... 2024-04-13T07:16:43.748 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-04-13T07:16:44.076 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-py. 2024-04-13T07:16:44.092 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../070-python3-py_1.10.0-1_all.deb ... 2024-04-13T07:16:44.126 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-py (1.10.0-1) ... 2024-04-13T07:16:44.446 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-iniconfig. 2024-04-13T07:16:44.473 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../071-python3-iniconfig_1.1.1-2_all.deb ... 2024-04-13T07:16:44.504 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-04-13T07:16:44.514 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-babel. 2024-04-13T07:16:44.541 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../065-python3-babel_2.8.0+dfsg.1-7_all.deb ... 2024-04-13T07:16:44.580 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-babel (2.8.0+dfsg.1-7) ... 2024-04-13T07:16:44.774 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-toml. 2024-04-13T07:16:44.801 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../072-python3-toml_0.10.2-1_all.deb ... 2024-04-13T07:16:44.832 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-04-13T07:16:44.917 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-jinja2. 2024-04-13T07:16:44.943 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../066-python3-jinja2_3.0.3-1ubuntu0.1_all.deb ... 2024-04-13T07:16:44.983 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-04-13T07:16:45.136 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pytest. 2024-04-13T07:16:45.162 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../073-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-04-13T07:16:45.202 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-04-13T07:16:45.370 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-cephadm. 2024-04-13T07:16:45.397 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../067-ceph-mgr-cephadm_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:45.436 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-cephadm (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:45.556 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-repoze.lru. 2024-04-13T07:16:45.583 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../074-python3-repoze.lru_0.7-2_all.deb ... 2024-04-13T07:16:45.623 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-04-13T07:16:45.774 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-bcrypt. 2024-04-13T07:16:45.800 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../068-python3-bcrypt_3.2.0-1build1_amd64.deb ... 2024-04-13T07:16:45.840 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-bcrypt (3.2.0-1build1) ... 2024-04-13T07:16:45.918 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-routes. 2024-04-13T07:16:45.944 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../075-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-04-13T07:16:45.984 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-04-13T07:16:46.118 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pluggy. 2024-04-13T07:16:46.145 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../069-python3-pluggy_0.13.0-7.1_all.deb ... 2024-04-13T07:16:46.185 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pluggy (0.13.0-7.1) ... 2024-04-13T07:16:46.330 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-04-13T07:16:46.356 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../076-ceph-mgr-dashboard_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:46.396 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-mgr-dashboard (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:46.463 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-py. 2024-04-13T07:16:46.490 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../070-python3-py_1.10.0-1_all.deb ... 2024-04-13T07:16:46.529 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-py (1.10.0-1) ... 2024-04-13T07:16:46.841 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-iniconfig. 2024-04-13T07:16:46.868 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../071-python3-iniconfig_1.1.1-2_all.deb ... 2024-04-13T07:16:46.908 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-iniconfig (1.1.1-2) ... 2024-04-13T07:16:47.211 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-toml. 2024-04-13T07:16:47.237 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../072-python3-toml_0.10.2-1_all.deb ... 2024-04-13T07:16:47.277 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-toml (0.10.2-1) ... 2024-04-13T07:16:47.580 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pytest. 2024-04-13T07:16:47.607 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../073-python3-pytest_6.2.5-1ubuntu2_all.deb ... 2024-04-13T07:16:47.646 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pytest (6.2.5-1ubuntu2) ... 2024-04-13T07:16:47.677 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-04-13T07:16:47.704 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../077-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-04-13T07:16:47.743 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-04-13T07:16:48.000 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-repoze.lru. 2024-04-13T07:16:48.027 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../074-python3-repoze.lru_0.7-2_all.deb ... 2024-04-13T07:16:48.067 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-repoze.lru (0.7-2) ... 2024-04-13T07:16:48.285 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-joblib. 2024-04-13T07:16:48.313 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../078-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-04-13T07:16:48.370 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-routes. 2024-04-13T07:16:48.396 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../075-python3-routes_2.5.1-1ubuntu1_all.deb ... 2024-04-13T07:16:48.436 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-routes (2.5.1-1ubuntu1) ... 2024-04-13T07:16:48.497 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-04-13T07:16:48.807 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-dashboard. 2024-04-13T07:16:48.834 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../076-ceph-mgr-dashboard_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:48.868 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-04-13T07:16:48.873 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-dashboard (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:48.895 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../079-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-04-13T07:16:48.934 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-04-13T07:16:49.229 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-sklearn. 2024-04-13T07:16:49.256 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../080-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-04-13T07:16:49.295 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-04-13T07:16:49.955 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-04-13T07:16:49.982 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../081-ceph-mgr-diskprediction-local_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:50.021 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-mgr-diskprediction-local (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:50.254 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-sklearn-lib:amd64. 2024-04-13T07:16:50.269 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../077-python3-sklearn-lib_0.23.2-5ubuntu6_amd64.deb ... 2024-04-13T07:16:50.303 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-04-13T07:16:50.807 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-joblib. 2024-04-13T07:16:50.834 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../078-python3-joblib_0.17.0-4ubuntu1_all.deb ... 2024-04-13T07:16:50.873 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-joblib (0.17.0-4ubuntu1) ... 2024-04-13T07:16:51.033 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-cachetools. 2024-04-13T07:16:51.060 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../082-python3-cachetools_5.0.0-1_all.deb ... 2024-04-13T07:16:51.099 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-04-13T07:16:51.244 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-threadpoolctl. 2024-04-13T07:16:51.271 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../079-python3-threadpoolctl_3.1.0-1_all.deb ... 2024-04-13T07:16:51.310 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-threadpoolctl (3.1.0-1) ... 2024-04-13T07:16:51.463 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pyasn1. 2024-04-13T07:16:51.490 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../083-python3-pyasn1_0.4.8-1_all.deb ... 2024-04-13T07:16:51.529 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-04-13T07:16:51.613 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-sklearn. 2024-04-13T07:16:51.640 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../080-python3-sklearn_0.23.2-5ubuntu6_all.deb ... 2024-04-13T07:16:51.680 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-sklearn (0.23.2-5ubuntu6) ... 2024-04-13T07:16:51.867 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-04-13T07:16:51.894 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../084-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-04-13T07:16:51.933 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-04-13T07:16:52.245 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-rsa. 2024-04-13T07:16:52.272 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../085-python3-rsa_4.8-1_all.deb ... 2024-04-13T07:16:52.311 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-rsa (4.8-1) ... 2024-04-13T07:16:52.364 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-diskprediction-local. 2024-04-13T07:16:52.391 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../081-ceph-mgr-diskprediction-local_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:52.430 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-diskprediction-local (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:52.598 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-google-auth. 2024-04-13T07:16:52.614 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../086-python3-google-auth_1.5.1-3_all.deb ... 2024-04-13T07:16:52.648 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-04-13T07:16:52.910 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-04-13T07:16:52.937 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../087-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-04-13T07:16:52.968 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-04-13T07:16:53.263 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-websocket. 2024-04-13T07:16:53.290 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../088-python3-websocket_1.2.3-1_all.deb ... 2024-04-13T07:16:53.330 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-04-13T07:16:53.379 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-cachetools. 2024-04-13T07:16:53.402 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../082-python3-cachetools_5.0.0-1_all.deb ... 2024-04-13T07:16:53.441 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-cachetools (5.0.0-1) ... 2024-04-13T07:16:53.650 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-kubernetes. 2024-04-13T07:16:53.677 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../089-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-04-13T07:16:53.708 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-04-13T07:16:53.813 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyasn1. 2024-04-13T07:16:53.840 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../083-python3-pyasn1_0.4.8-1_all.deb ... 2024-04-13T07:16:53.879 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyasn1 (0.4.8-1) ... 2024-04-13T07:16:54.217 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyasn1-modules. 2024-04-13T07:16:54.243 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../084-python3-pyasn1-modules_0.2.1-1_all.deb ... 2024-04-13T07:16:54.283 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyasn1-modules (0.2.1-1) ... 2024-04-13T07:16:54.339 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-04-13T07:16:54.368 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../090-ceph-mgr-k8sevents_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:54.406 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-mgr-k8sevents (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:54.611 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-rsa. 2024-04-13T07:16:54.638 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../085-python3-rsa_4.8-1_all.deb ... 2024-04-13T07:16:54.678 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-rsa (4.8-1) ... 2024-04-13T07:16:54.725 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libonig5:amd64. 2024-04-13T07:16:54.753 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../091-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-04-13T07:16:54.792 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-04-13T07:16:54.965 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-google-auth. 2024-04-13T07:16:54.989 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../086-python3-google-auth_1.5.1-3_all.deb ... 2024-04-13T07:16:55.023 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-google-auth (1.5.1-3) ... 2024-04-13T07:16:55.079 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libjq1:amd64. 2024-04-13T07:16:55.106 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../092-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-04-13T07:16:55.160 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-04-13T07:16:55.301 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-requests-oauthlib. 2024-04-13T07:16:55.328 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../087-python3-requests-oauthlib_1.3.0+ds-0.1_all.deb ... 2024-04-13T07:16:55.359 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-04-13T07:16:55.406 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package jq. 2024-04-13T07:16:55.434 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../093-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-04-13T07:16:55.473 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-04-13T07:16:55.671 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-websocket. 2024-04-13T07:16:55.698 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../088-python3-websocket_1.2.3-1_all.deb ... 2024-04-13T07:16:55.709 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package socat. 2024-04-13T07:16:55.728 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../094-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-04-13T07:16:55.737 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-websocket (1.2.3-1) ... 2024-04-13T07:16:55.767 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-04-13T07:16:56.057 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-kubernetes. 2024-04-13T07:16:56.084 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../089-python3-kubernetes_12.0.1-1ubuntu1_all.deb ... 2024-04-13T07:16:56.087 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package xmlstarlet. 2024-04-13T07:16:56.103 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../095-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-04-13T07:16:56.123 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-kubernetes (12.0.1-1ubuntu1) ... 2024-04-13T07:16:56.137 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-04-13T07:16:56.492 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-test. 2024-04-13T07:16:56.508 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../096-ceph-test_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:56.550 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-test (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:56.754 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-mgr-k8sevents. 2024-04-13T07:16:56.782 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../090-ceph-mgr-k8sevents_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:56.821 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-mgr-k8sevents (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:57.140 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libonig5:amd64. 2024-04-13T07:16:57.168 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../091-libonig5_6.9.7.1-2build1_amd64.deb ... 2024-04-13T07:16:57.207 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libonig5:amd64 (6.9.7.1-2build1) ... 2024-04-13T07:16:57.543 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libjq1:amd64. 2024-04-13T07:16:57.570 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../092-libjq1_1.6-2.1ubuntu3_amd64.deb ... 2024-04-13T07:16:57.610 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-04-13T07:16:57.846 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package jq. 2024-04-13T07:16:57.861 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../093-jq_1.6-2.1ubuntu3_amd64.deb ... 2024-04-13T07:16:57.895 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking jq (1.6-2.1ubuntu3) ... 2024-04-13T07:16:58.131 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package socat. 2024-04-13T07:16:58.147 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../094-socat_1.7.4.1-3ubuntu4_amd64.deb ... 2024-04-13T07:16:58.190 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking socat (1.7.4.1-3ubuntu4) ... 2024-04-13T07:16:58.493 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package xmlstarlet. 2024-04-13T07:16:58.509 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../095-xmlstarlet_1.6.1-2.1_amd64.deb ... 2024-04-13T07:16:58.551 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking xmlstarlet (1.6.1-2.1) ... 2024-04-13T07:16:58.747 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package ceph-volume. 2024-04-13T07:16:58.774 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../097-ceph-volume_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:16:58.813 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking ceph-volume (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:58.881 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-test. 2024-04-13T07:16:58.908 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../096-ceph-test_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:58.947 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-test (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:59.234 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package libcephfs-dev. 2024-04-13T07:16:59.250 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../098-libcephfs-dev_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:16:59.292 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking libcephfs-dev (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:16:59.671 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package nvme-cli. 2024-04-13T07:16:59.698 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../099-nvme-cli_1.16-3ubuntu0.1_amd64.deb ... 2024-04-13T07:16:59.737 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ... 2024-04-13T07:17:00.076 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-04-13T07:17:00.092 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../100-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-04-13T07:17:00.134 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-04-13T07:17:00.564 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pastescript. 2024-04-13T07:17:00.592 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../101-python3-pastescript_2.0.2-4_all.deb ... 2024-04-13T07:17:00.630 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-04-13T07:17:00.951 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-psutil. 2024-04-13T07:17:00.960 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package ceph-volume. 2024-04-13T07:17:00.979 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../102-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-04-13T07:17:00.987 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../097-ceph-volume_18.2.2-1217-g7d36322c-1jammy_all.deb ... 2024-04-13T07:17:01.017 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-04-13T07:17:01.026 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking ceph-volume (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:01.339 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package libcephfs-dev. 2024-04-13T07:17:01.366 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../098-libcephfs-dev_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:17:01.379 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pygments. 2024-04-13T07:17:01.405 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking libcephfs-dev (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:01.407 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../103-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-04-13T07:17:01.445 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-04-13T07:17:01.817 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package nvme-cli. 2024-04-13T07:17:01.844 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../099-nvme-cli_1.16-3ubuntu0.1_amd64.deb ... 2024-04-13T07:17:01.883 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking nvme-cli (1.16-3ubuntu0.1) ... 2024-04-13T07:17:01.925 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-pyinotify. 2024-04-13T07:17:01.953 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../104-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-04-13T07:17:01.991 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-04-13T07:17:02.247 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python-asyncssh-doc. 2024-04-13T07:17:02.263 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../100-python-asyncssh-doc_2.5.0-1_all.deb ... 2024-04-13T07:17:02.295 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package python3-simplejson. 2024-04-13T07:17:02.305 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python-asyncssh-doc (2.5.0-1) ... 2024-04-13T07:17:02.322 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../105-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-04-13T07:17:02.361 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-04-13T07:17:02.639 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-04-13T07:17:02.667 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../106-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-04-13T07:17:02.697 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-04-13T07:17:02.725 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pastescript. 2024-04-13T07:17:02.752 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../101-python3-pastescript_2.0.2-4_all.deb ... 2024-04-13T07:17:02.791 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pastescript (2.0.2-4) ... 2024-04-13T07:17:03.120 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-psutil. 2024-04-13T07:17:03.147 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../102-python3-psutil_5.9.0-1build1_amd64.deb ... 2024-04-13T07:17:03.186 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-psutil (5.9.0-1build1) ... 2024-04-13T07:17:03.469 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package radosgw. 2024-04-13T07:17:03.496 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../107-radosgw_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:17:03.534 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking radosgw (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:03.539 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pygments. 2024-04-13T07:17:03.566 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../103-python3-pygments_2.11.2+dfsg-2_all.deb ... 2024-04-13T07:17:03.614 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pygments (2.11.2+dfsg-2) ... 2024-04-13T07:17:04.085 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-pyinotify. 2024-04-13T07:17:04.112 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../104-python3-pyinotify_0.9.6-1.3_all.deb ... 2024-04-13T07:17:04.143 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-pyinotify (0.9.6-1.3) ... 2024-04-13T07:17:04.330 INFO:teuthology.orchestra.run.smithi203.stdout:Selecting previously unselected package rbd-fuse. 2024-04-13T07:17:04.358 INFO:teuthology.orchestra.run.smithi203.stdout:Preparing to unpack .../108-rbd-fuse_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:17:04.405 INFO:teuthology.orchestra.run.smithi203.stdout:Unpacking rbd-fuse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:04.446 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package python3-simplejson. 2024-04-13T07:17:04.473 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../105-python3-simplejson_3.17.6-1build1_amd64.deb ... 2024-04-13T07:17:04.512 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking python3-simplejson (3.17.6-1build1) ... 2024-04-13T07:17:04.734 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-04-13T07:17:04.782 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package qttranslations5-l10n. 2024-04-13T07:17:04.809 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../106-qttranslations5-l10n_5.15.3-1_all.deb ... 2024-04-13T07:17:04.848 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking qttranslations5-l10n (5.15.3-1) ... 2024-04-13T07:17:04.985 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-04-13T07:17:05.111 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ... 2024-04-13T07:17:05.408 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-04-13T07:17:05.561 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package radosgw. 2024-04-13T07:17:05.589 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../107-radosgw_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:17:05.644 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking radosgw (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:05.771 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-04-13T07:17:06.227 INFO:teuthology.orchestra.run.smithi203.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-04-13T07:17:06.346 INFO:teuthology.orchestra.run.smithi089.stdout:Selecting previously unselected package rbd-fuse. 2024-04-13T07:17:06.374 INFO:teuthology.orchestra.run.smithi089.stdout:Preparing to unpack .../108-rbd-fuse_18.2.2-1217-g7d36322c-1jammy_amd64.deb ... 2024-04-13T07:17:06.413 INFO:teuthology.orchestra.run.smithi089.stdout:Unpacking rbd-fuse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:06.514 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up cephadm (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:06.725 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-iniconfig (1.1.1-2) ... 2024-04-13T07:17:06.993 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libdouble-conversion3:amd64 (3.1.7-4) ... 2024-04-13T07:17:07.119 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up nvme-cli (1.16-3ubuntu0.1) ... 2024-04-13T07:17:07.156 INFO:teuthology.orchestra.run.smithi203.stdout:Adding system user cephadm....done 2024-04-13T07:17:07.338 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-04-13T07:17:07.384 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmefc-boot-connections.service -> /lib/systemd/system/nvmefc-boot-connections.service. 2024-04-13T07:17:07.622 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-04-13T07:17:07.697 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/default.target.wants/nvmf-autoconnect.service -> /lib/systemd/system/nvmf-autoconnect.service. 2024-04-13T07:17:07.865 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-04-13T07:17:08.008 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-04-13T07:17:08.121 INFO:teuthology.orchestra.run.smithi089.stdout:nvmf-connect.target is a disabled or a static unit, not starting it. 2024-04-13T07:17:08.239 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up cephadm (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:08.267 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-04-13T07:17:08.527 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-04-13T07:17:08.653 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-py (1.10.0-1) ... 2024-04-13T07:17:08.969 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-04-13T07:17:09.131 INFO:teuthology.orchestra.run.smithi089.stdout:Adding system user cephadm....done 2024-04-13T07:17:09.385 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-waitress (1.4.4-1.1ubuntu1) ... 2024-04-13T07:17:09.632 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-04-13T07:17:09.739 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jaraco.classes (3.2.1-3) ... 2024-04-13T07:17:09.900 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-04-13T07:17:10.007 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python-asyncssh-doc (2.5.0-1) ... 2024-04-13T07:17:10.133 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jaraco.functools (3.4.0-2) ... 2024-04-13T07:17:10.170 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-04-13T07:17:10.384 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-repoze.lru (0.7-2) ... 2024-04-13T07:17:10.419 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-04-13T07:17:10.635 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up liboath0:amd64 (2.6.7-3build1) ... 2024-04-13T07:17:10.687 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-04-13T07:17:10.753 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-py (1.10.0-1) ... 2024-04-13T07:17:10.980 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-ceph-argparse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:11.273 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-04-13T07:17:11.321 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-joblib (0.17.0-4ubuntu1) ... 2024-04-13T07:17:11.424 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-04-13T07:17:11.683 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-04-13T07:17:11.707 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cachetools (5.0.0-1) ... 2024-04-13T07:17:11.809 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-04-13T07:17:11.966 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-openssl (21.0.0-1) ... 2024-04-13T07:17:12.169 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-04-13T07:17:12.251 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-bcrypt (3.2.0-1build1) ... 2024-04-13T07:17:12.321 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-04-13T07:17:12.493 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyinotify (0.9.6-1.3) ... 2024-04-13T07:17:12.605 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-04-13T07:17:12.736 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-threadpoolctl (3.1.0-1) ... 2024-04-13T07:17:12.881 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-04-13T07:17:12.996 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-ceph-argparse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:13.208 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-04-13T07:17:13.280 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-sklearn-lib:amd64 (0.23.2-5ubuntu6) ... 2024-04-13T07:17:13.406 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-markupsafe (2.0.1-2build1) ... 2024-04-13T07:17:13.666 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libpcre2-16-0:amd64 (10.39-3ubuntu0.1) ... 2024-04-13T07:17:13.810 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-psutil (5.9.0-1build1) ... 2024-04-13T07:17:13.843 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-04-13T07:17:14.103 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-04-13T07:17:14.168 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python-babel-localedata (2.8.0+dfsg.1-7) ... 2024-04-13T07:17:14.229 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-04-13T07:17:14.277 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-natsort (8.0.2-1) ... 2024-04-13T07:17:14.380 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-04-13T07:17:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-routes (2.5.1-1ubuntu1) ... 2024-04-13T07:17:14.723 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-04-13T07:17:14.821 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-simplejson (3.17.6-1build1) ... 2024-04-13T07:17:15.116 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-04-13T07:17:15.122 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pygments (2.11.2+dfsg-2) ... 2024-04-13T07:17:15.359 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-04-13T07:17:15.752 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-04-13T07:17:15.799 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-tempita (0.5.2-6ubuntu1) ... 2024-04-13T07:17:16.054 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-04-13T07:17:16.067 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python-pastedeploy-tpl (2.1.1-1) ... 2024-04-13T07:17:16.185 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up qttranslations5-l10n (5.15.3-1) ... 2024-04-13T07:17:16.286 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-wcwidth (0.2.5+dfsg1-1) ... 2024-04-13T07:17:16.397 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-04-13T07:17:16.570 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-asyncssh (2.5.0-1) ... 2024-04-13T07:17:16.664 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-04-13T07:17:16.824 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-ceph-common (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:16.955 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-certifi (2020.6.20-1) ... 2024-04-13T07:17:17.117 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-idna (3.3-1) ... 2024-04-13T07:17:17.206 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-paste (3.5.0+dfsg1-1) ... 2024-04-13T07:17:17.435 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-04-13T07:17:17.591 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cheroot (8.5.2+ds1-1ubuntu3.1) ... 2024-04-13T07:17:17.912 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-werkzeug (2.0.2+dfsg1-1ubuntu0.22.04.1) ... 2024-04-13T07:17:18.260 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jaraco.text (3.6.0-2) ... 2024-04-13T07:17:18.520 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up socat (1.7.4.1-3ubuntu4) ... 2024-04-13T07:17:18.646 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-ceph-common (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:18.658 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:17:18.792 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-toml (0.10.2-1) ... 2024-04-13T07:17:18.955 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-idna (3.3-1) ... 2024-04-13T07:17:19.052 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-04-13T07:17:19.187 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-04-13T07:17:19.275 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-sklearn (0.23.2-5ubuntu6) ... 2024-04-13T07:17:19.296 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-04-13T07:17:19.572 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-04-13T07:17:19.840 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-04-13T07:17:20.092 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:17:20.218 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-04-13T07:17:20.344 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-04-13T07:17:20.448 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libqt5core5a:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:17:20.582 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-toml (0.10.2-1) ... 2024-04-13T07:17:20.662 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-04-13T07:17:20.858 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librdkafka1:amd64 (1.8.0-1build1) ... 2024-04-13T07:17:20.905 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-04-13T07:17:20.984 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up xmlstarlet (1.6.1-2.1) ... 2024-04-13T07:17:21.110 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-urllib3 (1.26.5-1~exp1ubuntu0.1) ... 2024-04-13T07:17:21.181 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-04-13T07:17:21.412 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pluggy (0.13.0-7.1) ... 2024-04-13T07:17:21.466 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-04-13T07:17:21.680 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-zc.lockfile (2.0-1) ... 2024-04-13T07:17:21.717 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-04-13T07:17:21.956 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libqt5dbus5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:17:21.984 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-04-13T07:17:22.098 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libfmt8:amd64 (8.1.1+ds1-2) ... 2024-04-13T07:17:22.119 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-04-13T07:17:22.250 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyasn1 (0.4.8-1) ... 2024-04-13T07:17:22.437 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-04-13T07:17:22.568 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-singledispatch (3.4.0.3-3) ... 2024-04-13T07:17:22.588 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-04-13T07:17:22.827 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-logutils (0.3.3-8) ... 2024-04-13T07:17:22.898 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-04-13T07:17:23.095 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-tempora (4.1.2-1) ... 2024-04-13T07:17:23.199 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-04-13T07:17:23.346 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-simplegeneric (0.8.1-3) ... 2024-04-13T07:17:23.467 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-04-13T07:17:23.597 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-prettytable (2.5.0-2) ... 2024-04-13T07:17:23.866 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up liblttng-ust1:amd64 (2.13.1-1ubuntu1) ... 2024-04-13T07:17:23.869 INFO:teuthology.orchestra.run.smithi203.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-04-13T07:17:24.008 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-websocket (1.2.3-1) ... 2024-04-13T07:17:24.137 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-04-13T07:17:24.238 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-04-13T07:17:24.284 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libonig5:amd64 (6.9.7.1-2build1) ... 2024-04-13T07:17:24.410 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-mako (1.1.3+ds1-2ubuntu0.1) ... 2024-04-13T07:17:24.712 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-webob (1:1.8.6-1.1) ... 2024-04-13T07:17:24.757 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-04-13T07:17:25.013 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jaraco.collections (3.4.0-2) ... 2024-04-13T07:17:25.116 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-portend (3.0.0-1) ... 2024-04-13T07:17:25.264 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-babel (2.8.0+dfsg.1-7) ... 2024-04-13T07:17:25.367 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:17:25.493 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-04-13T07:17:25.654 INFO:teuthology.orchestra.run.smithi089.stdout:update-alternatives: using /usr/bin/pybabel-python3 to provide /usr/bin/pybabel (pybabel) in auto mode 2024-04-13T07:17:25.820 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-04-13T07:17:25.900 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libjq1:amd64 (1.6-2.1ubuntu3) ... 2024-04-13T07:17:26.026 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pytest (6.2.5-1ubuntu2) ... 2024-04-13T07:17:26.163 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-04-13T07:17:26.444 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pastedeploy (2.1.1-1) ... 2024-04-13T07:17:26.447 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-04-13T07:17:26.598 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-rsa (4.8-1) ... 2024-04-13T07:17:26.854 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-portend (3.0.0-1) ... 2024-04-13T07:17:26.883 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-04-13T07:17:27.114 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libqt5network5:amd64 (5.15.3+dfsg-2ubuntu0.2) ... 2024-04-13T07:17:27.151 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-04-13T07:17:27.239 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-jinja2 (3.0.3-1ubuntu0.1) ... 2024-04-13T07:17:27.411 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-04-13T07:17:27.566 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pyasn1-modules (0.2.1-1) ... 2024-04-13T07:17:27.820 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-04-13T07:17:27.867 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-requests (2.25.1+dfsg-2ubuntu0.1) ... 2024-04-13T07:17:28.097 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-04-13T07:17:28.143 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up jq (1.6-2.1ubuntu3) ... 2024-04-13T07:17:28.269 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rsa (4.8-1) ... 2024-04-13T07:17:28.449 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-04-13T07:17:28.545 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-webtest (2.0.35-1) ... 2024-04-13T07:17:28.575 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up librados2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:28.701 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up librgw2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:28.822 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-requests-oauthlib (1.3.0+ds-0.1) ... 2024-04-13T07:17:28.836 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libsqlite3-mod-ceph (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:28.962 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libcephfs2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:29.063 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libradosstriper1 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:29.089 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cherrypy3 (18.6.1-4) ... 2024-04-13T07:17:29.267 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-04-13T07:17:29.483 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pastescript (2.0.2-4) ... 2024-04-13T07:17:29.543 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up librbd1 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:29.686 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-mgr-modules-core (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:29.792 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-pecan (1.3.3-4ubuntu2) ... 2024-04-13T07:17:29.837 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-fuse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:30.068 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-04-13T07:17:30.068 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-04-13T07:17:30.144 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libthrift-0.16.0:amd64 (0.16.0-2) ... 2024-04-13T07:17:30.304 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librados2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:30.447 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librgw2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:30.555 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up libcephfs-dev (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:30.573 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libsqlite3-mod-ceph (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:30.665 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-rados (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:30.699 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libcephfs2 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:30.791 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-04-13T07:17:30.833 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libradosstriper1 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:31.045 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-google-auth (1.5.1-3) ... 2024-04-13T07:17:31.322 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up librbd1 (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:31.448 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-modules-core (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:31.582 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-fuse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:31.817 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/remote-fs.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-04-13T07:17:31.817 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-fuse.target -> /lib/systemd/system/ceph-fuse.target. 2024-04-13T07:17:32.012 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-rbd (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:32.164 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up rbd-fuse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:32.258 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up libcephfs-dev (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:32.315 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-rgw (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:32.385 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rados (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:32.441 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up python3-cephfs (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:32.511 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-kubernetes (12.0.1-1ubuntu1) ... 2024-04-13T07:17:32.567 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-common (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:33.136 INFO:teuthology.orchestra.run.smithi203.stdout:Adding group ceph....done 2024-04-13T07:17:33.470 INFO:teuthology.orchestra.run.smithi203.stdout:Adding system user ceph....done 2024-04-13T07:17:33.689 INFO:teuthology.orchestra.run.smithi203.stdout:Setting system user ceph properties....done 2024-04-13T07:17:33.707 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rbd (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:33.709 INFO:teuthology.orchestra.run.smithi203.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-04-13T07:17:33.846 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-04-13T07:17:33.858 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up rbd-fuse (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:33.984 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-rgw (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:34.110 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up python3-cephfs (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:34.173 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-04-13T07:17:34.236 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-common (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:34.620 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-test (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:34.771 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up radosgw (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:34.796 INFO:teuthology.orchestra.run.smithi089.stdout:Adding group ceph....done 2024-04-13T07:17:35.148 INFO:teuthology.orchestra.run.smithi089.stdout:Adding system user ceph....done 2024-04-13T07:17:35.227 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-04-13T07:17:35.227 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-04-13T07:17:35.308 INFO:teuthology.orchestra.run.smithi089.stdout:Setting system user ceph properties....done 2024-04-13T07:17:35.327 INFO:teuthology.orchestra.run.smithi089.stdout:chown: cannot access '/var/log/ceph/*.log*': No such file or directory 2024-04-13T07:17:35.460 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /lib/systemd/system/ceph.target. 2024-04-13T07:17:35.681 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-base (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:35.756 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/rbdmap.service -> /lib/systemd/system/rbdmap.service. 2024-04-13T07:17:36.032 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-04-13T07:17:36.163 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-test (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:36.289 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up radosgw (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:36.500 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-mds (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:36.703 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-04-13T07:17:36.703 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-04-13T07:17:36.764 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-04-13T07:17:36.765 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-radosgw.target -> /lib/systemd/system/ceph-radosgw.target. 2024-04-13T07:17:37.160 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-mgr (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:37.199 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-base (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:37.401 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-04-13T07:17:37.401 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-04-13T07:17:37.547 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-crash.service -> /lib/systemd/system/ceph-crash.service. 2024-04-13T07:17:37.870 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-osd (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:38.010 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mds (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:38.298 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-04-13T07:17:38.298 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mds.target -> /lib/systemd/system/ceph-mds.target. 2024-04-13T07:17:38.306 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-04-13T07:17:38.306 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-04-13T07:17:38.788 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-mgr-k8sevents (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:38.821 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:38.939 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-mgr-diskprediction-local (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:39.037 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-04-13T07:17:39.038 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mgr.target -> /lib/systemd/system/ceph-mgr.target. 2024-04-13T07:17:39.090 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-mon (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:39.305 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-04-13T07:17:39.306 INFO:teuthology.orchestra.run.smithi203.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-04-13T07:17:39.505 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-osd (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:39.784 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-04-13T07:17:39.784 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-osd.target -> /lib/systemd/system/ceph-osd.target. 2024-04-13T07:17:39.817 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-mgr-cephadm (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:39.986 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:40.128 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-mgr-dashboard (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:40.327 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-k8sevents (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:40.454 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-diskprediction-local (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:40.458 INFO:teuthology.orchestra.run.smithi203.stdout:Setting up ceph-volume (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:40.605 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mon (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:40.819 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/multi-user.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-04-13T07:17:40.819 INFO:teuthology.orchestra.run.smithi089.stdout:Created symlink /etc/systemd/system/ceph.target.wants/ceph-mon.target -> /lib/systemd/system/ceph-mon.target. 2024-04-13T07:17:40.890 INFO:teuthology.orchestra.run.smithi203.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-04-13T07:17:41.339 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-cephadm (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:41.490 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:41.633 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-mgr-dashboard (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:41.784 INFO:teuthology.orchestra.run.smithi089.stdout:Setting up ceph-volume (18.2.2-1217-g7d36322c-1jammy) ... 2024-04-13T07:17:42.089 INFO:teuthology.orchestra.run.smithi203.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-04-13T07:17:42.196 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for man-db (2.10.2-1) ... 2024-04-13T07:17:42.578 INFO:teuthology.orchestra.run.smithi203.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-04-13T07:17:42.584 DEBUG:teuthology.parallel:result is None 2024-04-13T07:17:43.030 INFO:teuthology.orchestra.run.smithi089.stdout:Processing triggers for libc-bin (2.35-0ubuntu3.1) ... 2024-04-13T07:17:43.502 INFO:teuthology.orchestra.run.smithi089.stderr:W: --force-yes is deprecated, use one of the options starting with --allow instead. 2024-04-13T07:17:43.507 DEBUG:teuthology.parallel:result is None 2024-04-13T07:17:43.508 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-13T07:17:43.508 INFO:teuthology.packaging:ref: None 2024-04-13T07:17:43.508 INFO:teuthology.packaging:tag: None 2024-04-13T07:17:43.508 INFO:teuthology.packaging:branch: reef 2024-04-13T07:17:43.508 INFO:teuthology.packaging:sha1: 5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:17:43.508 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-04-13T07:17:43.799 DEBUG:teuthology.orchestra.run.smithi089:> dpkg-query -W -f '${Version}' ceph 2024-04-13T07:17:43.819 INFO:teuthology.orchestra.run.smithi089.stdout:18.2.2-1217-g7d36322c-1jammy 2024-04-13T07:17:43.819 INFO:teuthology.packaging:The installed version of ceph is 18.2.2-1217-g7d36322c-1jammy 2024-04-13T07:17:43.819 INFO:teuthology.task.install:The correct ceph version 18.2.2-1217-g7d36322c-1jammy is installed. 2024-04-13T07:17:43.821 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2024-04-13T07:17:43.821 INFO:teuthology.packaging:ref: None 2024-04-13T07:17:43.821 INFO:teuthology.packaging:tag: None 2024-04-13T07:17:43.821 INFO:teuthology.packaging:branch: reef 2024-04-13T07:17:43.821 INFO:teuthology.packaging:sha1: 5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:17:43.821 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&ref=reef 2024-04-13T07:17:44.101 DEBUG:teuthology.orchestra.run.smithi203:> dpkg-query -W -f '${Version}' ceph 2024-04-13T07:17:44.121 INFO:teuthology.orchestra.run.smithi203.stdout:18.2.2-1217-g7d36322c-1jammy 2024-04-13T07:17:44.121 INFO:teuthology.packaging:The installed version of ceph is 18.2.2-1217-g7d36322c-1jammy 2024-04-13T07:17:44.121 INFO:teuthology.task.install:The correct ceph version 18.2.2-1217-g7d36322c-1jammy is installed. 2024-04-13T07:17:44.123 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2024-04-13T07:17:44.123 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:17:44.123 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-13T07:17:44.136 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:17:44.137 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2024-04-13T07:17:44.177 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2024-04-13T07:17:44.177 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:17:44.178 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/usr/bin/daemon-helper 2024-04-13T07:17:44.191 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-13T07:17:44.245 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:17:44.245 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/usr/bin/daemon-helper 2024-04-13T07:17:44.258 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod a=rx -- /usr/bin/daemon-helper 2024-04-13T07:17:44.310 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2024-04-13T07:17:44.311 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:17:44.311 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-13T07:17:44.323 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-13T07:17:44.377 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:17:44.377 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/usr/bin/adjust-ulimits 2024-04-13T07:17:44.389 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2024-04-13T07:17:44.442 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2024-04-13T07:17:44.443 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:17:44.443 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/usr/bin/stdin-killer 2024-04-13T07:17:44.455 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-13T07:17:44.509 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:17:44.509 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/usr/bin/stdin-killer 2024-04-13T07:17:44.522 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod a=rx -- /usr/bin/stdin-killer 2024-04-13T07:17:44.575 INFO:teuthology.run_tasks:Running task cephadm... 2024-04-13T07:17:44.677 INFO:tasks.cephadm:Config: {'compiled_cephadm_branch': 'reef', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:reef', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '5f22a4974d6492df1d519913f5251731f043c582'} 2024-04-13T07:17:44.677 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:reef 2024-04-13T07:17:44.678 INFO:tasks.cephadm:Cluster fsid is ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:17:44.678 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2024-04-13T07:17:44.678 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.89', 'mon.c': '[v2:172.21.15.89:3301,v1:172.21.15.89:6790]', 'mon.b': '172.21.15.203'} 2024-04-13T07:17:44.679 INFO:tasks.cephadm:First mon is mon.a on smithi089 2024-04-13T07:17:44.679 INFO:tasks.cephadm:First mgr is y 2024-04-13T07:17:44.679 INFO:tasks.cephadm:Normalizing hostnames... 2024-04-13T07:17:44.679 DEBUG:teuthology.orchestra.run.smithi089:> sudo hostname $(hostname -s) 2024-04-13T07:17:44.693 DEBUG:teuthology.orchestra.run.smithi203:> sudo hostname $(hostname -s) 2024-04-13T07:17:44.707 INFO:tasks.cephadm:Downloading "compiled" cephadm from cachra for reef 2024-04-13T07:17:44.707 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=ubuntu%2F22.04%2Fx86_64&sha1=5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:17:44.924 INFO:tasks.cephadm:builder_project result: [{'status': 'ready', 'sha1': '5f22a4974d6492df1d519913f5251731f043c582', 'extra': {'build_url': 'https://jenkins.ceph.com/job/ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=jammy,DIST=jammy,MACHINE_SIZE=gigantic/54881/', 'root_build_cause': 'SCMTRIGGER', 'version': '19.0.0-2905-g5f22a497', 'node_name': '172.21.2.17+braggi17', 'job_name': 'ceph-dev-build/ARCH=x86_64,AVAILABLE_ARCH=x86_64,AVAILABLE_DIST=jammy,DIST=jammy,MACHINE_SIZE=gigantic', 'package_manager_version': '19.0.0-2905-g5f22a497-1jammy'}, 'url': 'https://2.chacra.ceph.com/r/ceph/main/5f22a4974d6492df1d519913f5251731f043c582/ubuntu/jammy/flavors/default/', 'distro_codename': 'jammy', 'modified': '2024-04-12 17:24:15.997063', 'distro_version': '22.04', 'project': 'ceph', 'flavor': 'default', 'ref': 'main', 'chacra_url': 'https://2.chacra.ceph.com/repos/ceph/main/5f22a4974d6492df1d519913f5251731f043c582/ubuntu/jammy/flavors/default/', 'archs': ['x86_64'], 'distro': 'ubuntu'}] 2024-04-13T07:17:45.206 INFO:tasks.util.chacra:got chacra host 1.chacra.ceph.com, ref reef, sha1 7d36322c373c73b184fd252cd881b122d8c51a3b from https://shaman.ceph.com/api/search/?project=ceph&distros=ubuntu%2F22.04%2Fx86_64&flavor=default&ref=reef 2024-04-13T07:17:45.208 INFO:tasks.cephadm:Discovered cachra url: https://1.chacra.ceph.com/binaries/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/x86_64/flavors/default/cephadm 2024-04-13T07:17:45.208 DEBUG:teuthology.orchestra.run.smithi089:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-13T07:17:45.869 INFO:teuthology.orchestra.run.smithi089.stdout:-rw-rw-r-- 1 ubuntu ubuntu 216842 Apr 13 07:17 /home/ubuntu/cephtest/cephadm 2024-04-13T07:17:45.870 DEBUG:teuthology.orchestra.run.smithi203:> curl --silent -L https://1.chacra.ceph.com/binaries/ceph/reef/7d36322c373c73b184fd252cd881b122d8c51a3b/ubuntu/jammy/x86_64/flavors/default/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2024-04-13T07:17:46.369 INFO:teuthology.orchestra.run.smithi203.stdout:-rw-rw-r-- 1 ubuntu ubuntu 216842 Apr 13 07:17 /home/ubuntu/cephtest/cephadm 2024-04-13T07:17:46.369 DEBUG:teuthology.orchestra.run.smithi089:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-13T07:17:46.377 DEBUG:teuthology.orchestra.run.smithi203:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2024-04-13T07:17:46.398 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:reef on all hosts... 2024-04-13T07:17:46.398 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-04-13T07:17:46.423 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef pull 2024-04-13T07:17:46.543 INFO:teuthology.orchestra.run.smithi089.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-04-13T07:17:46.547 INFO:teuthology.orchestra.run.smithi203.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-04-13T07:19:03.875 INFO:teuthology.orchestra.run.smithi203.stdout:{ 2024-04-13T07:19:03.875 INFO:teuthology.orchestra.run.smithi203.stdout: "ceph_version": "ceph version 18.2.2-1217-g7d36322c (7d36322c373c73b184fd252cd881b122d8c51a3b) reef (stable)", 2024-04-13T07:19:03.876 INFO:teuthology.orchestra.run.smithi203.stdout: "image_id": "6ecd1226aa2fbfe626757c10af365cae7e1efb29811ef174caf43c1f61820704", 2024-04-13T07:19:03.876 INFO:teuthology.orchestra.run.smithi203.stdout: "repo_digests": [ 2024-04-13T07:19:03.876 INFO:teuthology.orchestra.run.smithi203.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:7c6481ef05df1ed65f50090b5bbf17a24dc65f935462074ab3ceb527e874a60d" 2024-04-13T07:19:03.876 INFO:teuthology.orchestra.run.smithi203.stdout: ] 2024-04-13T07:19:03.876 INFO:teuthology.orchestra.run.smithi203.stdout:} 2024-04-13T07:19:05.073 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-04-13T07:19:05.074 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph_version": "ceph version 18.2.2-1217-g7d36322c (7d36322c373c73b184fd252cd881b122d8c51a3b) reef (stable)", 2024-04-13T07:19:05.074 INFO:teuthology.orchestra.run.smithi089.stdout: "image_id": "6ecd1226aa2fbfe626757c10af365cae7e1efb29811ef174caf43c1f61820704", 2024-04-13T07:19:05.074 INFO:teuthology.orchestra.run.smithi089.stdout: "repo_digests": [ 2024-04-13T07:19:05.074 INFO:teuthology.orchestra.run.smithi089.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:7c6481ef05df1ed65f50090b5bbf17a24dc65f935462074ab3ceb527e874a60d" 2024-04-13T07:19:05.074 INFO:teuthology.orchestra.run.smithi089.stdout: ] 2024-04-13T07:19:05.074 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-04-13T07:19:05.099 DEBUG:teuthology.orchestra.run.smithi089:> sudo mkdir -p /etc/ceph 2024-04-13T07:19:05.113 DEBUG:teuthology.orchestra.run.smithi203:> sudo mkdir -p /etc/ceph 2024-04-13T07:19:05.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 777 /etc/ceph 2024-04-13T07:19:05.171 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod 777 /etc/ceph 2024-04-13T07:19:05.183 INFO:tasks.cephadm:Writing seed config... 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [osd] debug ms = 1 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [osd] debug osd = 20 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2024-04-13T07:19:05.185 INFO:tasks.cephadm: override: [mon] debug mon = 20 2024-04-13T07:19:05.186 INFO:tasks.cephadm: override: [mon] debug ms = 1 2024-04-13T07:19:05.186 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2024-04-13T07:19:05.186 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:19:05.186 DEBUG:teuthology.orchestra.run.smithi089:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2024-04-13T07:19:05.218 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log to file = true mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = ec50a262-f965-11ee-bc8d-c7b262605968 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2024-04-13T07:19:05.219 DEBUG:teuthology.orchestra.run.smithi089:mon.a> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a.service 2024-04-13T07:19:05.262 DEBUG:teuthology.orchestra.run.smithi089:mgr.y> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y.service 2024-04-13T07:19:05.305 INFO:tasks.cephadm:Bootstrapping... 2024-04-13T07:19:05.306 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef -v bootstrap --fsid ec50a262-f965-11ee-bc8d-c7b262605968 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.89 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2024-04-13T07:19:05.473 INFO:teuthology.orchestra.run.smithi089.stdout:-------------------------------------------------------------------------------- 2024-04-13T07:19:05.473 INFO:teuthology.orchestra.run.smithi089.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:reef', '-v', 'bootstrap', '--fsid', 'ec50a262-f965-11ee-bc8d-c7b262605968', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.89', '--skip-admin-label'] 2024-04-13T07:19:05.474 INFO:teuthology.orchestra.run.smithi089.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2024-04-13T07:19:05.474 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying podman|docker is present... 2024-04-13T07:19:05.475 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying lvm2 is present... 2024-04-13T07:19:05.475 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying time synchronization is in place... 2024-04-13T07:19:05.481 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-13T07:19:05.481 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-13T07:19:05.484 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-13T07:19:05.485 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-04-13T07:19:05.488 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-04-13T07:19:05.488 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-04-13T07:19:05.492 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-04-13T07:19:05.492 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-04-13T07:19:05.495 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-04-13T07:19:05.495 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout masked 2024-04-13T07:19:05.499 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-04-13T07:19:05.499 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-04-13T07:19:05.502 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-04-13T07:19:05.502 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-04-13T07:19:05.506 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-04-13T07:19:05.506 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-04-13T07:19:05.510 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout enabled 2024-04-13T07:19:05.514 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout active 2024-04-13T07:19:05.514 INFO:teuthology.orchestra.run.smithi089.stdout:Unit ntp.service is enabled and running 2024-04-13T07:19:05.514 INFO:teuthology.orchestra.run.smithi089.stdout:Repeating the final host check... 2024-04-13T07:19:05.514 INFO:teuthology.orchestra.run.smithi089.stdout:docker (/usr/bin/docker) is present 2024-04-13T07:19:05.514 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl is present 2024-04-13T07:19:05.514 INFO:teuthology.orchestra.run.smithi089.stdout:lvcreate is present 2024-04-13T07:19:05.517 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2024-04-13T07:19:05.517 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2024-04-13T07:19:05.520 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2024-04-13T07:19:05.520 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-04-13T07:19:05.524 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled chronyd.service 2024-04-13T07:19:05.524 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for chronyd.service: No such file or directory 2024-04-13T07:19:05.527 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active chronyd.service 2024-04-13T07:19:05.527 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-04-13T07:19:05.530 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled systemd-timesyncd.service 2024-04-13T07:19:05.530 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout masked 2024-04-13T07:19:05.533 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active systemd-timesyncd.service 2024-04-13T07:19:05.533 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-04-13T07:19:05.536 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl is-enabled ntpd.service 2024-04-13T07:19:05.537 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to get unit file state for ntpd.service: No such file or directory 2024-04-13T07:19:05.540 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 3 from systemctl is-active ntpd.service 2024-04-13T07:19:05.540 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout inactive 2024-04-13T07:19:05.543 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout enabled 2024-04-13T07:19:05.547 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stdout active 2024-04-13T07:19:05.547 INFO:teuthology.orchestra.run.smithi089.stdout:Unit ntp.service is enabled and running 2024-04-13T07:19:05.547 INFO:teuthology.orchestra.run.smithi089.stdout:Host looks OK 2024-04-13T07:19:05.547 INFO:teuthology.orchestra.run.smithi089.stdout:Cluster fsid: ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:05.548 INFO:teuthology.orchestra.run.smithi089.stdout:Acquiring lock 140381755606496 on /run/cephadm/ec50a262-f965-11ee-bc8d-c7b262605968.lock 2024-04-13T07:19:05.548 INFO:teuthology.orchestra.run.smithi089.stdout:Lock 140381755606496 acquired on /run/cephadm/ec50a262-f965-11ee-bc8d-c7b262605968.lock 2024-04-13T07:19:05.548 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying IP 172.21.15.89 port 3300 ... 2024-04-13T07:19:05.548 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying IP 172.21.15.89 port 6789 ... 2024-04-13T07:19:05.548 INFO:teuthology.orchestra.run.smithi089.stdout:Base mon IP(s) is [172.21.15.89:3300, 172.21.15.89:6789], mon addrv is [v2:172.21.15.89:3300,v1:172.21.15.89:6789] 2024-04-13T07:19:05.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 2024-04-13T07:19:05.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.89 metric 100 2024-04-13T07:19:05.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 158.69.67.47 via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.89 metric 100 2024-04-13T07:19:05.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.17.0.0/16 dev docker0 proto kernel scope link src 172.17.0.1 linkdown 2024-04-13T07:19:05.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.89 metric 100 2024-04-13T07:19:05.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.21.0.1 dev enp3s0f1 proto dhcp scope link src 172.21.15.89 metric 100 2024-04-13T07:19:05.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.21.0.2 dev enp3s0f1 proto dhcp scope link src 172.21.15.89 metric 100 2024-04-13T07:19:05.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 172.21.15.254 dev enp3s0f1 proto dhcp scope link src 172.21.15.89 metric 100 2024-04-13T07:19:05.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2024-04-13T07:19:05.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2024-04-13T07:19:05.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 100 expires 1764sec pref medium 2024-04-13T07:19:05.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2024-04-13T07:19:05.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout inet6 ::1/128 scope host 2024-04-13T07:19:05.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-13T07:19:05.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2024-04-13T07:19:05.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d501/64 scope link 2024-04-13T07:19:05.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/sbin/ip: stdout valid_lft forever preferred_lft forever 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.0/20` 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.0/20` 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.1/32` 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.1/32` 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.2/32` 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.0.2/32` 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.15.254/32` 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Mon IP `172.21.15.89` is in CIDR network `172.21.15.254/32` 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20', '172.21.0.1/32', '172.21.0.1/32', '172.21.0.2/32', '172.21.0.2/32', '172.21.15.254/32', '172.21.15.254/32'] 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2024-04-13T07:19:05.556 INFO:teuthology.orchestra.run.smithi089.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:reef... 2024-04-13T07:19:05.790 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/docker: stdout reef: Pulling from ceph-ci/ceph 2024-04-13T07:19:05.790 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/docker: stdout Digest: sha256:7c6481ef05df1ed65f50090b5bbf17a24dc65f935462074ab3ceb527e874a60d 2024-04-13T07:19:05.791 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/docker: stdout Status: Image is up to date for quay.ceph.io/ceph-ci/ceph:reef 2024-04-13T07:19:05.791 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/docker: stdout quay.ceph.io/ceph-ci/ceph:reef 2024-04-13T07:19:07.162 INFO:teuthology.orchestra.run.smithi089.stdout:ceph: stdout ceph version 18.2.2-1217-g7d36322c (7d36322c373c73b184fd252cd881b122d8c51a3b) reef (stable) 2024-04-13T07:19:07.162 INFO:teuthology.orchestra.run.smithi089.stdout:Ceph version: ceph version 18.2.2-1217-g7d36322c (7d36322c373c73b184fd252cd881b122d8c51a3b) reef (stable) 2024-04-13T07:19:07.162 INFO:teuthology.orchestra.run.smithi089.stdout:Extracting ceph user uid/gid from container image... 2024-04-13T07:19:08.349 INFO:teuthology.orchestra.run.smithi089.stdout:stat: stdout 167 167 2024-04-13T07:19:08.349 INFO:teuthology.orchestra.run.smithi089.stdout:Creating initial keys... 2024-04-13T07:19:09.514 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-authtool: stdout AQDtMRpmHWUUARAARQWNxJIuaIu2056+x4zYfA== 2024-04-13T07:19:10.734 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-authtool: stdout AQDuMRpm8wWSDBAAKC4Mj/rYc/0+R/AIaQN50A== 2024-04-13T07:19:11.972 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-authtool: stdout AQDvMRpmkszFFhAAjG9NVJ/Se6bKQs5JlI9o1Q== 2024-04-13T07:19:11.973 INFO:teuthology.orchestra.run.smithi089.stdout:Creating initial monmap... 2024-04-13T07:19:13.186 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = pacific 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout:monmaptool for a [v2:172.21.15.89:3300,v1:172.21.15.89:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout:setting min_mon_release = pacific 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: set fsid to ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:19:13.187 INFO:teuthology.orchestra.run.smithi089.stdout:Creating mon... 2024-04-13T07:19:14.536 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 1 imported monmap: 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr epoch 0 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr last_changed 2024-04-13T07:19:12.689346+0000 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr min_mon_release 16 (pacific) 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr election_strategy: 1 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 0 /usr/bin/ceph-mon: set fsid to ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: RocksDB version: 7.9.2 2024-04-13T07:19:14.537 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Git sha 0 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Compile date 2024-04-12 16:22:45 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: DB SUMMARY 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: DB Session ID: 7IC1H3H78ROQ0HL6X674 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 0, files: 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.error_if_exists: 0 2024-04-13T07:19:14.538 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.create_if_missing: 1 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.paranoid_checks: 1 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.env: 0x559a10d46820 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.fs: PosixFileSystem 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.info_log: 0x559a13240780 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.statistics: (nil) 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.use_fsync: 0 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_log_file_size: 0 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-04-13T07:19:14.539 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.allow_fallocate: 1 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.use_direct_reads: 0 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.create_missing_column_families: 0 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.db_log_dir: 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.wal_dir: 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-13T07:19:14.540 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.advise_random_on_open: 1 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.write_buffer_manager: 0x559a124d5180 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.rate_limiter: (nil) 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.unordered_write: 0 2024-04-13T07:19:14.541 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.row_cache: None 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.wal_filter: None 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.two_write_queues: 0 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.manual_wal_flush: 0 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.wal_compression: 0 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.atomic_flush: 0 2024-04-13T07:19:14.542 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.log_readahead_size: 0 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_background_jobs: 2 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_background_compactions: -1 2024-04-13T07:19:14.543 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_subcompactions: 1 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_total_wal_size: 0 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_open_files: -1 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.bytes_per_sync: 0 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-04-13T07:19:14.544 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Options.max_background_flushes: -1 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Compression algorithms supported: 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: kZSTD supported: 0 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: kXpressCompression supported: 0 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: kZlibCompression supported: 1 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: kSnappyCompression supported: 1 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: kLZ4Compression supported: 1 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: kBZip2Compression supported: 0 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.965+0000 7f67087e4c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: [db/db_impl/db_impl_open.cc:317] Creating manifest 1 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-13T07:19:14.545 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.merge_operator: 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_filter: None 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_filter_factory: None 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x559a132408a0) 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks: 1 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr cache_index_and_filter_blocks_with_high_priority: 0 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr pin_top_level_index_and_filter: 1 2024-04-13T07:19:14.546 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr index_type: 0 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr data_block_index_type: 0 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr index_shortening: 1 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr data_block_hash_table_util_ratio: 0.750000 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr checksum: 4 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr no_block_cache: 0 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_cache: 0x559a125551f0 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_cache_name: BinnedLRUCache 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_cache_options: 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr capacity : 536870912 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr num_shard_bits : 4 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr strict_capacity_limit : 0 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr high_pri_pool_ratio: 0.000 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_cache_compressed: (nil) 2024-04-13T07:19:14.547 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr persistent_cache: (nil) 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_size: 4096 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_size_deviation: 10 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_restart_interval: 16 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr index_block_restart_interval: 1 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr metadata_block_size: 4096 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr partition_filters: 0 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr use_delta_encoding: 1 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr filter_policy: bloomfilter 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr whole_key_filtering: 1 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr verify_compression: 0 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr read_amp_bytes_per_bit: 0 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr format_version: 5 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr enable_index_compression: 1 2024-04-13T07:19:14.548 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr block_align: 0 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr max_auto_readahead_size: 262144 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr prepopulate_block_cache: 0 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr initial_auto_readahead_size: 8192 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr num_file_reads_for_auto_readahead: 2 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression: NoCompression 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.num_levels: 7 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-13T07:19:14.549 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.level: 32767 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-04-13T07:19:14.550 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.enabled: false 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-13T07:19:14.551 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.arena_block_size: 1048576 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-13T07:19:14.552 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.table_properties_collectors: 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.inplace_update_support: 0 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-13T07:19:14.553 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.bloom_locality: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.max_successive_merges: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.force_consistency_checks: 1 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.ttl: 2592000 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.enable_blob_files: false 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.min_blob_size: 0 2024-04-13T07:19:14.554 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.blob_file_size: 268435456 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000001 succeeded,manifest_file_number is 1, next_file_number is 3, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 0 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ce66e667-534b-4b0f-bd19-3c033b687d2c 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 5 2024-04-13T07:19:14.555 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x559a125ce000 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f67087e4c80 4 rocksdb: DB pointer 0x559a125c0000 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f66f89c2700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.969+0000 7f66f89c2700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** DB Stats ** 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.556 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Sum 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** Compaction Stats [default] ** 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.557 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Flush(GB): cumulative 0.000, interval 0.000 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(GB): cumulative 0.000, interval 0.000 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(Total Files): cumulative 0, interval 0 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(L0 Files): cumulative 0, interval 0 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr AddFile(Keys): cumulative 0, interval 0 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Cumulative compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Block cache BinnedLRUCache@0x559a125551f0#7 capacity: 512.00 MB usage: 0.00 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr Block cache entry stats(count,size,portion): Misc(1,0.00 KB,0%) 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr ** File Read Latency Histogram By Level [default] ** 2024-04-13T07:19:14.558 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr 2024-04-13T07:19:14.559 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.973+0000 7f67087e4c80 4 rocksdb: [db/db_impl/db_impl.cc:496] Shutdown: canceling all background work 2024-04-13T07:19:14.559 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.973+0000 7f67087e4c80 4 rocksdb: [db/db_impl/db_impl.cc:704] Shutdown complete 2024-04-13T07:19:14.559 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph-mon: stderr debug 2024-04-13T07:19:13.973+0000 7f67087e4c80 0 /usr/bin/ceph-mon: created monfs at /var/lib/ceph/mon/ceph-a for mon.a 2024-04-13T07:19:14.559 INFO:teuthology.orchestra.run.smithi089.stdout:create mon.a on 2024-04-13T07:19:14.791 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2024-04-13T07:19:14.997 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target -> /etc/systemd/system/ceph.target. 2024-04-13T07:19:15.210 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-ec50a262-f965-11ee-bc8d-c7b262605968.target -> /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968.target. 2024-04-13T07:19:15.211 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-ec50a262-f965-11ee-bc8d-c7b262605968.target -> /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968.target. 2024-04-13T07:19:15.460 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a 2024-04-13T07:19:15.460 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to reset failed state of unit ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a.service: Unit ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a.service not loaded. 2024-04-13T07:19:15.662 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968.target.wants/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a.service -> /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service. 2024-04-13T07:19:15.670 INFO:teuthology.orchestra.run.smithi089.stdout:firewalld does not appear to be present 2024-04-13T07:19:15.670 INFO:teuthology.orchestra.run.smithi089.stdout:Not possible to enable service . firewalld.service is not available 2024-04-13T07:19:15.670 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mon to start... 2024-04-13T07:19:15.670 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mon... 2024-04-13T07:19:17.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:16 smithi089 bash[20865]: cluster 2024-04-13T07:19:16.911388+0000 mon.a (mon.0) 1 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-04-13T07:19:17.861 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout cluster: 2024-04-13T07:19:17.861 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout id: ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:17.861 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2024-04-13T07:19:17.861 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout services: 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.337075s) 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout data: 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout pgs: 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:mon is available 2024-04-13T07:19:17.862 INFO:teuthology.orchestra.run.smithi089.stdout:Assimilating anything we can from ceph.conf... 2024-04-13T07:19:18.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:17 smithi089 bash[20865]: cluster 2024-04-13T07:19:16.935343+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-04-13T07:19:18.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:17 smithi089 bash[20865]: cluster 2024-04-13T07:19:16.935555+0000 mon.a (mon.0) 3 : cluster 0 monmap e1: 1 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-13T07:19:18.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:17 smithi089 bash[20865]: cluster 2024-04-13T07:19:16.937789+0000 mon.a (mon.0) 4 : cluster 0 fsmap 2024-04-13T07:19:18.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:17 smithi089 bash[20865]: cluster 2024-04-13T07:19:16.940092+0000 mon.a (mon.0) 5 : cluster 0 osdmap e1: 0 total, 0 up, 0 in 2024-04-13T07:19:18.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:17 smithi089 bash[20865]: cluster 2024-04-13T07:19:16.940498+0000 mon.a (mon.0) 6 : cluster 0 mgrmap e1: no daemons active 2024-04-13T07:19:18.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:17 smithi089 bash[20865]: audit 2024-04-13T07:19:17.272383+0000 mon.a (mon.0) 7 : audit 0 from='client.? 172.21.15.89:0/434809279' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2024-04-13T07:19:19.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:18 smithi089 bash[20865]: audit 2024-04-13T07:19:18.859367+0000 mon.a (mon.0) 8 : audit 1 from='client.? 172.21.15.89:0/2856813719' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-13T07:19:19.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:18 smithi089 bash[20865]: audit 2024-04-13T07:19:18.864454+0000 mon.a (mon.0) 9 : audit 1 from='client.? 172.21.15.89:0/2856813719' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2024-04-13T07:19:19.358 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [global] 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout fsid = ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.89:3300,v1:172.21.15.89:6789] 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:19.359 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [osd] 2024-04-13T07:19:19.360 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-13T07:19:19.360 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-13T07:19:19.360 INFO:teuthology.orchestra.run.smithi089.stdout:Generating new minimal ceph.conf... 2024-04-13T07:19:20.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:20 smithi089 bash[20865]: audit 2024-04-13T07:19:20.310226+0000 mon.a (mon.0) 10 : audit 0 from='client.? 172.21.15.89:0/3380705813' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:19:20.845 INFO:teuthology.orchestra.run.smithi089.stdout:Restarting the monitor... 2024-04-13T07:19:21.163 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:20 smithi089 systemd[1]: Stopping Ceph mon.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:19:21.163 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:20 smithi089 bash[20865]: debug 2024-04-13T07:19:20.984+0000 7f31c47e5700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-13T07:19:21.163 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:20 smithi089 bash[20865]: debug 2024-04-13T07:19:20.984+0000 7f31c47e5700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2024-04-13T07:19:21.709 INFO:teuthology.orchestra.run.smithi089.stdout:Setting public_network to 172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32 in mon config section 2024-04-13T07:19:21.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:21 smithi089 bash[21253]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mon-a 2024-04-13T07:19:21.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:21 smithi089 bash[21314]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mon-a 2024-04-13T07:19:21.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:21 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-13T07:19:21.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:21 smithi089 systemd[1]: Stopped Ceph mon.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:19:21.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:21 smithi089 systemd[1]: Started Ceph mon.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:19:23.027 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.688+0000 7f1459068c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.688+0000 7f1459068c80 0 ceph version 18.2.2-1217-g7d36322c (7d36322c373c73b184fd252cd881b122d8c51a3b) reef (stable), process ceph-mon, pid 7 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.688+0000 7f1459068c80 0 pidfile_write: ignore empty --pid-file 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 0 load: jerasure load: lrc 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: RocksDB version: 7.9.2 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Git sha 0 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Compile date 2024-04-12 16:22:45 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: DB SUMMARY 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: DB Session ID: KE8C37T6173H17KJA6EV 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: CURRENT file: CURRENT 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: IDENTITY file: IDENTITY 2024-04-13T07:19:23.028 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 179 Bytes 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000009.log size: 84538 ; 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.error_if_exists: 0 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.create_if_missing: 0 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.paranoid_checks: 1 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.env: 0x561921b37820 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.fs: PosixFileSystem 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.info_log: 0x561924f29560 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-04-13T07:19:23.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.statistics: (nil) 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.use_fsync: 0 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_log_file_size: 0 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.allow_fallocate: 1 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.use_direct_reads: 0 2024-04-13T07:19:23.030 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.create_missing_column_families: 0 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.db_log_dir: 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.wal_dir: 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.advise_random_on_open: 1 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.write_buffer_manager: 0x5619241ba320 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-13T07:19:23.031 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-13T07:19:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-04-13T07:19:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.rate_limiter: (nil) 2024-04-13T07:19:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-13T07:19:23.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-04-13T07:19:23.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-04-13T07:19:23.033 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.unordered_write: 0 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.row_cache: None 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.wal_filter: None 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.two_write_queues: 0 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.manual_wal_flush: 0 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.wal_compression: 0 2024-04-13T07:19:23.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.atomic_flush: 0 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.log_readahead_size: 0 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_background_jobs: 2 2024-04-13T07:19:23.035 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_background_compactions: -1 2024-04-13T07:19:23.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_subcompactions: 1 2024-04-13T07:19:23.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-13T07:19:23.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-13T07:19:23.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-04-13T07:19:23.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_total_wal_size: 0 2024-04-13T07:19:23.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-13T07:19:23.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-04-13T07:19:23.036 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_open_files: -1 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bytes_per_sync: 0 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_background_flushes: -1 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Compression algorithms supported: 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-13T07:19:23.037 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: kZSTD supported: 0 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: kXpressCompression supported: 0 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: kZlibCompression supported: 1 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: kSnappyCompression supported: 1 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: kLZ4Compression supported: 1 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: kBZip2Compression supported: 0 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.merge_operator: 2024-04-13T07:19:23.038 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_filter: None 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_filter_factory: None 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561924f29660) 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: cache_index_and_filter_blocks: 1 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: pin_top_level_index_and_filter: 1 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: index_type: 0 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: data_block_index_type: 0 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: index_shortening: 1 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: data_block_hash_table_util_ratio: 0.750000 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: checksum: 4 2024-04-13T07:19:23.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: no_block_cache: 0 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: block_cache: 0x56192423d1f0 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: block_cache_name: BinnedLRUCache 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: block_cache_options: 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: capacity : 536870912 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: num_shard_bits : 4 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: strict_capacity_limit : 0 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: high_pri_pool_ratio: 0.000 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: block_cache_compressed: (nil) 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: persistent_cache: (nil) 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: block_size: 4096 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: block_size_deviation: 10 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: block_restart_interval: 16 2024-04-13T07:19:23.040 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: index_block_restart_interval: 1 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: metadata_block_size: 4096 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: partition_filters: 0 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: use_delta_encoding: 1 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: filter_policy: bloomfilter 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: whole_key_filtering: 1 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: verify_compression: 0 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: read_amp_bytes_per_bit: 0 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: format_version: 5 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: enable_index_compression: 1 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: block_align: 0 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: max_auto_readahead_size: 262144 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: prepopulate_block_cache: 0 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: initial_auto_readahead_size: 8192 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: num_file_reads_for_auto_readahead: 2 2024-04-13T07:19:23.041 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression: NoCompression 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.num_levels: 7 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-13T07:19:23.042 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.level: 32767 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.enabled: false 2024-04-13T07:19:23.043 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-13T07:19:23.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.arena_block_size: 1048576 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-13T07:19:23.045 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.table_properties_collectors: 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.inplace_update_support: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.bloom_locality: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.max_successive_merges: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.force_consistency_checks: 1 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-04-13T07:19:23.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.ttl: 2592000 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.enable_blob_files: false 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.min_blob_size: 0 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.blob_file_size: 268435456 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 12, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 5 2024-04-13T07:19:23.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 5 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ce66e667-534b-4b0f-bd19-3c033b687d2c 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712992762702183, "job": 1, "event": "recovery_started", "wal_files": [9]} 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.696+0000 7f1459068c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #9 mode 2 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.700+0000 7f1459068c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712992762703719, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 13, "file_size": 81062, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 8, "largest_seqno": 256, "table_properties": {"data_size": 79238, "index_size": 234, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10267, "raw_average_key_size": 47, "raw_value_size": 73439, "raw_average_value_size": 341, "num_data_blocks": 10, "num_entries": 215, "num_filter_entries": 215, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712992762, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ce66e667-534b-4b0f-bd19-3c033b687d2c", "db_session_id": "KE8C37T6173H17KJA6EV", "orig_file_number": 13, "seqno_to_time_mapping": "N/A"}} 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.700+0000 7f1459068c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712992762703854, "job": 1, "event": "recovery_finished"} 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.700+0000 7f1459068c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 15 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.700+0000 7f1459068c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000009.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.700+0000 7f1459068c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5619242e0000 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.700+0000 7f1459068c80 4 rocksdb: DB pointer 0x5619242ce000 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.700+0000 7f1447840700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.700+0000 7f1447840700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: ** DB Stats ** 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-13T07:19:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: ** Compaction Stats [default] ** 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: L0 2/0 81.00 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 81.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Sum 2/0 81.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 81.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 81.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: ** Compaction Stats [default] ** 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 81.2 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-13T07:19:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Flush(GB): cumulative 0.000, interval 0.000 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: AddFile(Total Files): cumulative 0, interval 0 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: AddFile(L0 Files): cumulative 0, interval 0 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: AddFile(Keys): cumulative 0, interval 0 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Cumulative compaction: 0.00 GB write, 15.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Interval compaction: 0.00 GB write, 15.46 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Block cache BinnedLRUCache@0x56192423d1f0#7 capacity: 512.00 MB usage: 1.19 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: Block cache entry stats(count,size,portion): FilterBlock(2,0.77 KB,0.000146031%) IndexBlock(2,0.42 KB,8.04663e-05%) Misc(1,0.00 KB,0%) 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: ** File Read Latency Histogram By Level [default] ** 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] at bind addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:23.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 1 mon.a@-1(???) e1 preinit fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 0 mon.a@-1(???).mds e1 new map 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 0 mon.a@-1(???).mds e1 print_map 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: e1 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: enable_multiple, ever_enabled_multiple: 1,1 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: legacy client fscid: -1 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: No filesystems configured 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 0 mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 0 mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: debug 2024-04-13T07:19:22.704+0000 7f1459068c80 1 mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: cluster 2024-04-13T07:19:22.731472+0000 mon.a (mon.0) 1 : cluster 1 mon.a is new leader, mons a in quorum (ranks 0) 2024-04-13T07:19:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: cluster 2024-04-13T07:19:22.731589+0000 mon.a (mon.0) 2 : cluster 0 monmap e1: 1 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-13T07:19:23.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: cluster 2024-04-13T07:19:22.732020+0000 mon.a (mon.0) 3 : cluster 0 fsmap 2024-04-13T07:19:23.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: cluster 2024-04-13T07:19:22.732061+0000 mon.a (mon.0) 4 : cluster 0 osdmap e1: 0 total, 0 up, 0 in 2024-04-13T07:19:23.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:22 smithi089 bash[21353]: cluster 2024-04-13T07:19:22.732497+0000 mon.a (mon.0) 5 : cluster 0 mgrmap e1: no daemons active 2024-04-13T07:19:23.572 INFO:teuthology.orchestra.run.smithi089.stdout:Wrote config to /etc/ceph/ceph.conf 2024-04-13T07:19:23.598 INFO:teuthology.orchestra.run.smithi089.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2024-04-13T07:19:23.598 INFO:teuthology.orchestra.run.smithi089.stdout:Creating mgr... 2024-04-13T07:19:23.598 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying port 0.0.0.0:9283 ... 2024-04-13T07:19:23.598 INFO:teuthology.orchestra.run.smithi089.stdout:Verifying port 0.0.0.0:8765 ... 2024-04-13T07:19:23.923 INFO:teuthology.orchestra.run.smithi089.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y 2024-04-13T07:19:23.923 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Failed to reset failed state of unit ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y.service: Unit ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y.service not loaded. 2024-04-13T07:19:24.012 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:19:24.117 INFO:teuthology.orchestra.run.smithi089.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968.target.wants/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y.service -> /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service. 2024-04-13T07:19:24.124 INFO:teuthology.orchestra.run.smithi089.stdout:firewalld does not appear to be present 2024-04-13T07:19:24.124 INFO:teuthology.orchestra.run.smithi089.stdout:Not possible to enable service . firewalld.service is not available 2024-04-13T07:19:24.124 INFO:teuthology.orchestra.run.smithi089.stdout:firewalld does not appear to be present 2024-04-13T07:19:24.124 INFO:teuthology.orchestra.run.smithi089.stdout:Not possible to open ports <[9283, 8765]>. firewalld.service is not available 2024-04-13T07:19:24.124 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mgr to start... 2024-04-13T07:19:24.125 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mgr... 2024-04-13T07:19:24.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:24 smithi089 bash[21353]: audit 2024-04-13T07:19:23.016150+0000 mon.a (mon.0) 6 : audit 1 from='client.? 172.21.15.89:0/1831000371' entity='client.admin' 2024-04-13T07:19:24.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:24 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:19:25.792 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:25 smithi089 bash[21353]: audit 2024-04-13T07:19:25.555432+0000 mon.a (mon.0) 7 : audit 0 from='client.? 172.21.15.89:0/3182303554' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-13T07:19:25.793 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:25 smithi089 bash[21606]: debug 2024-04-13T07:19:25.480+0000 7f2b7034c200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:19:26.145 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsid": "ec50a262-f965-11ee-bc8d-c7b262605968", 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "health": { 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 0 2024-04-13T07:19:26.146 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "a" 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-13T07:19:26.147 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-13T07:19:26.148 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-13T07:19:26.149 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "restful" 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modified": "2024-04-13T07:19:16.936723+0000", 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-04-13T07:19:26.150 INFO:teuthology.orchestra.run.smithi089.stdout:mgr not available, waiting (1/15)... 2024-04-13T07:19:27.303 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:27 smithi089 bash[21606]: debug 2024-04-13T07:19:27.043+0000 7f2b7034c200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:19:27.303 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:27 smithi089 bash[21606]: debug 2024-04-13T07:19:27.135+0000 7f2b7034c200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:19:27.803 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:27 smithi089 bash[21606]: debug 2024-04-13T07:19:27.379+0000 7f2b7034c200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:19:28.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:28 smithi089 bash[21606]: debug 2024-04-13T07:19:28.163+0000 7f2b7034c200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:19:28.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:28 smithi089 bash[21606]: debug 2024-04-13T07:19:28.275+0000 7f2b7034c200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:19:28.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:28 smithi089 bash[21606]: debug 2024-04-13T07:19:28.375+0000 7f2b7034c200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:19:29.053 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:28 smithi089 bash[21606]: debug 2024-04-13T07:19:28.727+0000 7f2b7034c200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:19:29.053 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:28 smithi089 bash[21606]: debug 2024-04-13T07:19:28.839+0000 7f2b7034c200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:19:29.053 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:28 smithi089 bash[21606]: debug 2024-04-13T07:19:28.947+0000 7f2b7034c200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:19:29.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:29 smithi089 bash[21353]: audit 2024-04-13T07:19:29.157669+0000 mon.a (mon.0) 8 : audit 0 from='client.? 172.21.15.89:0/2127239592' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-13T07:19:29.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:29 smithi089 bash[21606]: debug 2024-04-13T07:19:29.307+0000 7f2b7034c200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsid": "ec50a262-f965-11ee-bc8d-c7b262605968", 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "health": { 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-13T07:19:29.671 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 0 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "a" 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_age": 6, 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-13T07:19:29.672 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-13T07:19:29.677 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-13T07:19:29.677 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-13T07:19:29.677 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-13T07:19:29.677 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:29.677 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-13T07:19:29.677 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-13T07:19:29.677 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-13T07:19:29.677 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-13T07:19:29.678 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "restful" 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modified": "2024-04-13T07:19:16.936723+0000", 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-04-13T07:19:29.679 INFO:teuthology.orchestra.run.smithi089.stdout:mgr not available, waiting (2/15)... 2024-04-13T07:19:29.803 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:29 smithi089 bash[21606]: debug 2024-04-13T07:19:29.547+0000 7f2b7034c200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:19:30.189 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:29 smithi089 bash[21606]: debug 2024-04-13T07:19:29.807+0000 7f2b7034c200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:19:30.190 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:29 smithi089 bash[21606]: debug 2024-04-13T07:19:29.915+0000 7f2b7034c200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:19:30.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:30 smithi089 bash[21606]: debug 2024-04-13T07:19:30.183+0000 7f2b7034c200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:19:30.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:30 smithi089 bash[21606]: debug 2024-04-13T07:19:30.423+0000 7f2b7034c200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:19:31.188 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:30 smithi089 bash[21606]: debug 2024-04-13T07:19:30.827+0000 7f2b7034c200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:19:31.188 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:30 smithi089 bash[21606]: debug 2024-04-13T07:19:30.943+0000 7f2b7034c200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:19:31.188 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:31 smithi089 bash[21606]: debug 2024-04-13T07:19:31.183+0000 7f2b7034c200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:19:32.553 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:32 smithi089 bash[21606]: debug 2024-04-13T07:19:32.179+0000 7f2b7034c200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:19:32.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:32 smithi089 bash[21606]: debug 2024-04-13T07:19:32.291+0000 7f2b7034c200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:19:32.874 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:32 smithi089 bash[21353]: audit 2024-04-13T07:19:32.658767+0000 mon.a (mon.0) 9 : audit 0 from='client.? 172.21.15.89:0/1138457853' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-13T07:19:32.874 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:32 smithi089 bash[21606]: debug 2024-04-13T07:19:32.595+0000 7f2b7034c200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:19:32.875 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:32 smithi089 bash[21606]: debug 2024-04-13T07:19:32.707+0000 7f2b7034c200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:19:33.183 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:33.183 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-04-13T07:19:33.183 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsid": "ec50a262-f965-11ee-bc8d-c7b262605968", 2024-04-13T07:19:33.183 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "health": { 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 0 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "a" 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_age": 9, 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-13T07:19:33.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-13T07:19:33.185 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:33.185 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-13T07:19:33.185 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:33.185 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-13T07:19:33.185 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-13T07:19:33.185 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-13T07:19:33.185 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-13T07:19:33.188 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-13T07:19:33.188 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-13T07:19:33.188 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:33.188 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:33.189 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": false, 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "restful" 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modified": "2024-04-13T07:19:16.936723+0000", 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:33.190 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-13T07:19:33.191 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-04-13T07:19:33.191 INFO:teuthology.orchestra.run.smithi089.stdout:mgr not available, waiting (3/15)... 2024-04-13T07:19:33.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21606]: debug 2024-04-13T07:19:33.151+0000 7f2b7034c200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:19:33.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21606]: debug 2024-04-13T07:19:33.275+0000 7f2b7034c200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:19:34.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: cluster 2024-04-13T07:19:33.282639+0000 mon.a (mon.0) 10 : cluster 1 Activating manager daemon y 2024-04-13T07:19:34.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: cluster 2024-04-13T07:19:33.286631+0000 mon.a (mon.0) 11 : cluster 0 mgrmap e2: y(active, starting, since 0.00433093s) 2024-04-13T07:19:34.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.289694+0000 mon.a (mon.0) 12 : audit 0 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:19:34.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.289947+0000 mon.a (mon.0) 13 : audit 0 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:19:34.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.290123+0000 mon.a (mon.0) 14 : audit 0 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:19:34.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.290285+0000 mon.a (mon.0) 15 : audit 0 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:19:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.290704+0000 mon.a (mon.0) 16 : audit 0 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:19:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: cluster 2024-04-13T07:19:33.317093+0000 mon.a (mon.0) 17 : cluster 1 Manager daemon y is now available 2024-04-13T07:19:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.328960+0000 mon.a (mon.0) 18 : audit 1 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:19:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.331607+0000 mon.a (mon.0) 19 : audit 1 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:19:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.334886+0000 mon.a (mon.0) 20 : audit 1 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' 2024-04-13T07:19:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.337094+0000 mon.a (mon.0) 21 : audit 1 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' 2024-04-13T07:19:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:33 smithi089 bash[21353]: audit 2024-04-13T07:19:33.339152+0000 mon.a (mon.0) 22 : audit 1 from='mgr.14100 172.21.15.89:0/1436501603' entity='mgr.y' 2024-04-13T07:19:35.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:35 smithi089 bash[21353]: cluster 2024-04-13T07:19:34.297398+0000 mon.a (mon.0) 23 : cluster 0 mgrmap e3: y(active, since 1.01508s) 2024-04-13T07:19:37.115 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:37.115 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsid": "ec50a262-f965-11ee-bc8d-c7b262605968", 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "health": { 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "checks": {}, 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mutes": [] 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum": [ 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 0 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "a" 2024-04-13T07:19:37.116 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "quorum_age": 13, 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "monmap": { 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "reef", 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_mons": 1 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osdmap": { 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2024-04-13T07:19:37.117 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgmap": { 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:37.121 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "fsmap": { 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "by_rank": [], 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "up:standby": 0 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap": { 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modules": [ 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "iostat", 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "nfs", 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "restful" 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ], 2024-04-13T07:19:37.122 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "servicemap": { 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 1, 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "modified": "2024-04-13T07:19:16.936723+0000", 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "services": {} 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout }, 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "progress_events": {} 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-04-13T07:19:37.123 INFO:teuthology.orchestra.run.smithi089.stdout:mgr is available 2024-04-13T07:19:37.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:37 smithi089 bash[21353]: cluster 2024-04-13T07:19:36.295828+0000 mon.a (mon.0) 24 : cluster 0 mgrmap e4: y(active, since 3s) 2024-04-13T07:19:37.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:37 smithi089 bash[21353]: audit 2024-04-13T07:19:36.599802+0000 mon.a (mon.0) 25 : audit 0 from='client.? 172.21.15.89:0/4017271840' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2024-04-13T07:19:38.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:38 smithi089 bash[21353]: audit 2024-04-13T07:19:38.369586+0000 mon.a (mon.0) 26 : audit 1 from='client.? 172.21.15.89:0/237036298' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2024-04-13T07:19:38.978 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:38.978 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [global] 2024-04-13T07:19:38.978 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout fsid = ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:38.978 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2024-04-13T07:19:38.978 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [mgr] 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout [osd] 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2024-04-13T07:19:38.980 INFO:teuthology.orchestra.run.smithi089.stdout:Enabling cephadm module... 2024-04-13T07:19:40.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:40 smithi089 bash[21353]: audit 2024-04-13T07:19:40.297967+0000 mon.a (mon.0) 27 : audit 1 from='client.? 172.21.15.89:0/2967728634' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2024-04-13T07:19:40.804 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:40 smithi089 bash[21606]: ignoring --setuser ceph since I am not root 2024-04-13T07:19:40.805 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:40 smithi089 bash[21606]: ignoring --setgroup ceph since I am not root 2024-04-13T07:19:40.805 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:40 smithi089 bash[21606]: debug 2024-04-13T07:19:40.534+0000 7fab47126200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:19:41.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:41 smithi089 bash[21353]: audit 2024-04-13T07:19:40.354978+0000 mon.a (mon.0) 28 : audit 1 from='client.? 172.21.15.89:0/2967728634' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2024-04-13T07:19:41.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:41 smithi089 bash[21353]: cluster 2024-04-13T07:19:40.355057+0000 mon.a (mon.0) 29 : cluster 0 mgrmap e5: y(active, since 7s) 2024-04-13T07:19:42.397 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:42 smithi089 bash[21353]: audit 2024-04-13T07:19:42.202145+0000 mon.a (mon.0) 30 : audit 0 from='client.? 172.21.15.89:0/434628208' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-13T07:19:42.397 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:42 smithi089 bash[21606]: debug 2024-04-13T07:19:42.110+0000 7fab47126200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:19:42.397 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:42 smithi089 bash[21606]: debug 2024-04-13T07:19:42.210+0000 7fab47126200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:19:42.705 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-04-13T07:19:42.705 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 5, 2024-04-13T07:19:42.705 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-13T07:19:42.705 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-13T07:19:42.705 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-13T07:19:42.705 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-04-13T07:19:42.706 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for the mgr to restart... 2024-04-13T07:19:42.706 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mgr epoch 5... 2024-04-13T07:19:42.722 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:42 smithi089 bash[21606]: debug 2024-04-13T07:19:42.462+0000 7fab47126200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:19:43.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:43 smithi089 bash[21606]: debug 2024-04-13T07:19:43.258+0000 7fab47126200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:19:43.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:43 smithi089 bash[21606]: debug 2024-04-13T07:19:43.374+0000 7fab47126200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:19:43.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:43 smithi089 bash[21606]: debug 2024-04-13T07:19:43.482+0000 7fab47126200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:19:44.304 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:43 smithi089 bash[21606]: debug 2024-04-13T07:19:43.834+0000 7fab47126200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:19:44.304 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:43 smithi089 bash[21606]: debug 2024-04-13T07:19:43.950+0000 7fab47126200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:19:44.304 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:44 smithi089 bash[21606]: debug 2024-04-13T07:19:44.050+0000 7fab47126200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:19:44.804 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:44 smithi089 bash[21606]: debug 2024-04-13T07:19:44.402+0000 7fab47126200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:19:44.804 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:44 smithi089 bash[21606]: debug 2024-04-13T07:19:44.638+0000 7fab47126200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:19:45.282 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:44 smithi089 bash[21606]: debug 2024-04-13T07:19:44.898+0000 7fab47126200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:19:45.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:45 smithi089 bash[21606]: debug 2024-04-13T07:19:45.010+0000 7fab47126200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:19:45.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:45 smithi089 bash[21606]: debug 2024-04-13T07:19:45.278+0000 7fab47126200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:19:45.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:45 smithi089 bash[21606]: debug 2024-04-13T07:19:45.518+0000 7fab47126200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:19:46.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:45 smithi089 bash[21606]: debug 2024-04-13T07:19:45.918+0000 7fab47126200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:19:46.283 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:46 smithi089 bash[21606]: debug 2024-04-13T07:19:46.038+0000 7fab47126200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:19:46.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:46 smithi089 bash[21606]: debug 2024-04-13T07:19:46.282+0000 7fab47126200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:19:47.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:47 smithi089 bash[21606]: debug 2024-04-13T07:19:47.274+0000 7fab47126200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:19:47.555 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:47 smithi089 bash[21606]: debug 2024-04-13T07:19:47.390+0000 7fab47126200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:19:48.054 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:47 smithi089 bash[21606]: debug 2024-04-13T07:19:47.678+0000 7fab47126200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:19:48.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:47 smithi089 bash[21606]: debug 2024-04-13T07:19:47.790+0000 7fab47126200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:19:48.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21606]: debug 2024-04-13T07:19:48.238+0000 7fab47126200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:19:48.554 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21606]: debug 2024-04-13T07:19:48.354+0000 7fab47126200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: cluster 2024-04-13T07:19:48.376801+0000 mon.a (mon.0) 31 : cluster 1 Active manager daemon y restarted 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: cluster 2024-04-13T07:19:48.377503+0000 mon.a (mon.0) 32 : cluster 1 Activating manager daemon y 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: cluster 2024-04-13T07:19:48.387350+0000 mon.a (mon.0) 33 : cluster 0 osdmap e2: 0 total, 0 up, 0 in 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: cluster 2024-04-13T07:19:48.387668+0000 mon.a (mon.0) 34 : cluster 0 mgrmap e6: y(active, starting, since 0.01047s) 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: audit 2024-04-13T07:19:48.388963+0000 mon.a (mon.0) 35 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: audit 2024-04-13T07:19:48.391333+0000 mon.a (mon.0) 36 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: audit 2024-04-13T07:19:48.392649+0000 mon.a (mon.0) 37 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: audit 2024-04-13T07:19:48.393025+0000 mon.a (mon.0) 38 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: audit 2024-04-13T07:19:48.393222+0000 mon.a (mon.0) 39 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:19:48.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:48 smithi089 bash[21353]: cluster 2024-04-13T07:19:48.417299+0000 mon.a (mon.0) 40 : cluster 1 Manager daemon y is now available 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: cephadm 2024-04-13T07:19:48.430950+0000 mgr.y (mgr.14120) 1 : cephadm 1 Found migration_current of "None". Setting to last migration. 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: audit 2024-04-13T07:19:48.441004+0000 mon.a (mon.0) 41 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: audit 2024-04-13T07:19:48.447218+0000 mon.a (mon.0) 42 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: audit 2024-04-13T07:19:48.451437+0000 mon.a (mon.0) 43 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: audit 2024-04-13T07:19:48.454839+0000 mon.a (mon.0) 44 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: audit 2024-04-13T07:19:48.465236+0000 mon.a (mon.0) 45 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: audit 2024-04-13T07:19:48.467974+0000 mon.a (mon.0) 46 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: audit 2024-04-13T07:19:49.387184+0000 mgr.y (mgr.14120) 2 : audit 0 from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: cluster 2024-04-13T07:19:49.390471+0000 mon.a (mon.0) 47 : cluster 0 mgrmap e7: y(active, since 1.01326s) 2024-04-13T07:19:49.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:49 smithi089 bash[21353]: audit 2024-04-13T07:19:49.401339+0000 mgr.y (mgr.14120) 3 : audit 0 from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-13T07:19:50.015 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-04-13T07:19:50.016 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2024-04-13T07:19:50.016 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-13T07:19:50.016 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-04-13T07:19:50.016 INFO:teuthology.orchestra.run.smithi089.stdout:mgr epoch 5 is available 2024-04-13T07:19:50.016 INFO:teuthology.orchestra.run.smithi089.stdout:Setting orchestrator backend to cephadm... 2024-04-13T07:19:51.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:50 smithi089 bash[21353]: audit 2024-04-13T07:19:49.776121+0000 mon.a (mon.0) 48 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:19:51.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:50 smithi089 bash[21353]: audit 2024-04-13T07:19:49.781147+0000 mon.a (mon.0) 49 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:19:51.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:50 smithi089 bash[21353]: cephadm 2024-04-13T07:19:49.892869+0000 mgr.y (mgr.14120) 4 : cephadm 1 [13/Apr/2024:07:19:49] ENGINE Bus STARTING 2024-04-13T07:19:51.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:50 smithi089 bash[21353]: cephadm 2024-04-13T07:19:49.994906+0000 mgr.y (mgr.14120) 5 : cephadm 1 [13/Apr/2024:07:19:49] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:19:51.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:50 smithi089 bash[21353]: cephadm 2024-04-13T07:19:50.112076+0000 mgr.y (mgr.14120) 6 : cephadm 1 [13/Apr/2024:07:19:50] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:19:51.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:50 smithi089 bash[21353]: cephadm 2024-04-13T07:19:50.112440+0000 mgr.y (mgr.14120) 7 : cephadm 1 [13/Apr/2024:07:19:50] ENGINE Bus STARTED 2024-04-13T07:19:51.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:50 smithi089 bash[21353]: audit 2024-04-13T07:19:50.126734+0000 mon.a (mon.0) 50 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:19:52.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:51 smithi089 bash[21353]: cluster 2024-04-13T07:19:50.781839+0000 mon.a (mon.0) 51 : cluster 0 mgrmap e8: y(active, since 2s) 2024-04-13T07:19:52.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:51 smithi089 bash[21353]: audit 2024-04-13T07:19:51.294032+0000 mgr.y (mgr.14120) 8 : audit 0 from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:19:52.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:51 smithi089 bash[21353]: audit 2024-04-13T07:19:51.304615+0000 mon.a (mon.0) 52 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:19:52.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:51 smithi089 bash[21353]: audit 2024-04-13T07:19:51.348657+0000 mon.a (mon.0) 53 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:19:53.580 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout value unchanged 2024-04-13T07:19:53.580 INFO:teuthology.orchestra.run.smithi089.stdout:Generating ssh key... 2024-04-13T07:19:53.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:53 smithi089 bash[21353]: audit 2024-04-13T07:19:53.056233+0000 mgr.y (mgr.14120) 9 : audit 0 from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:19:55.305 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: Generating public/private rsa key pair. 2024-04-13T07:19:55.305 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: Your identification has been saved in /tmp/tmpzckgrsbs/key. 2024-04-13T07:19:55.305 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: Your public key has been saved in /tmp/tmpzckgrsbs/key.pub. 2024-04-13T07:19:55.305 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: The key fingerprint is: 2024-04-13T07:19:55.305 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: SHA256:yFe6bsDUU3eKvJnqLPErg081AOmq34o/rKesGEURW44 ceph-ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:55.305 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: The key's randomart image is: 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: +---[RSA 3072]----+ 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: | o.o. | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: | *.. . . . | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: | E.. .. o.o o | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: | . ...oooo . | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: | .. oo S. + | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: | .. +o o+ | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: |.o ..+.. | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: |+o+...+o+ | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: |BB+o...*=. | 2024-04-13T07:19:55.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:19:54 smithi089 bash[21606]: +----[SHA256]-----+ 2024-04-13T07:19:56.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:56 smithi089 bash[21353]: audit 2024-04-13T07:19:54.814763+0000 mgr.y (mgr.14120) 10 : audit 0 from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:19:56.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:56 smithi089 bash[21353]: cephadm 2024-04-13T07:19:54.815214+0000 mgr.y (mgr.14120) 11 : cephadm 1 Generating ssh key... 2024-04-13T07:19:56.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:56 smithi089 bash[21353]: audit 2024-04-13T07:19:55.004445+0000 mon.a (mon.0) 54 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:19:56.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:56 smithi089 bash[21353]: audit 2024-04-13T07:19:55.008008+0000 mon.a (mon.0) 55 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:19:57.173 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWKiuga8eDTQ0XIPnQBkZwFvLzJ75lrKoVzJEqwvE4xNYihT75pT7iZyntoAKDZTYhmQ9Rk99YtD2MienXEPMIrJgvqn+qkTu8cuc0G4LMqhcGOdl3ub7yKG+6lhA72SMZEA/TVPmgs74PMXQ3OWNVCXRvzBEAso5eDr8gYKECGgIHgow013L8LdE+b+9hUg1CQ9Idwsn3jjUm967l558DQny8SsnnaddtSzavHFI2xZh3NLpClOmaRD3XM8FjcW36tGwzrGkMz0QyIH+DnaMqYWBswk7+3GxtoDoBWRLFhMPHNk1FA8c7Z0FNZszCKV+MspxFv3F9+yYKMVtT2MNGhhoLnpeGA7nnp/HuPZGOSK/gDUUvhLD6co4WF437MOmVUe74lx4Dv57u+6fD905cYz9WBDBYjl6etzcbmYhvlukNkiX5tMOR9AEISSwkFY8AqMShyA3yeeXv5SykHO10U8qgMi3lMDcw1bcUPnDab6uel2FG/fa32/n55dce7cU= ceph-ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:19:57.173 INFO:teuthology.orchestra.run.smithi089.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2024-04-13T07:19:57.174 INFO:teuthology.orchestra.run.smithi089.stdout:Adding key to root@localhost authorized_keys... 2024-04-13T07:19:57.174 INFO:teuthology.orchestra.run.smithi089.stdout:Adding host smithi089... 2024-04-13T07:19:57.788 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:57 smithi089 bash[21353]: audit 2024-04-13T07:19:56.727831+0000 mgr.y (mgr.14120) 12 : audit 0 from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:19:58.669 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:58 smithi089 bash[21353]: audit 2024-04-13T07:19:58.369186+0000 mgr.y (mgr.14120) 13 : audit 0 from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi089", "addr": "172.21.15.89", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:19:59.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:19:59 smithi089 bash[21353]: cephadm 2024-04-13T07:19:58.965061+0000 mgr.y (mgr.14120) 14 : cephadm 1 Deploying cephadm binary to smithi089 2024-04-13T07:20:02.184 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout Added host 'smithi089' with addr '172.21.15.89' 2024-04-13T07:20:02.184 INFO:teuthology.orchestra.run.smithi089.stdout:Deploying unmanaged mon service... 2024-04-13T07:20:02.771 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:02 smithi089 bash[21353]: audit 2024-04-13T07:20:01.331187+0000 mon.a (mon.0) 56 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:20:02.771 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:02 smithi089 bash[21353]: cephadm 2024-04-13T07:20:01.331867+0000 mgr.y (mgr.14120) 15 : cephadm 1 Added host smithi089 2024-04-13T07:20:02.771 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:02 smithi089 bash[21353]: audit 2024-04-13T07:20:01.350268+0000 mon.a (mon.0) 57 : audit 0 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:20:04.263 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2024-04-13T07:20:04.263 INFO:teuthology.orchestra.run.smithi089.stdout:Deploying unmanaged mgr service... 2024-04-13T07:20:04.930 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:04 smithi089 bash[21353]: audit 2024-04-13T07:20:03.674549+0000 mon.a (mon.0) 58 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:20:04.930 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:04 smithi089 bash[21353]: audit 2024-04-13T07:20:03.681044+0000 mgr.y (mgr.14120) 16 : audit 0 from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:20:04.930 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:04 smithi089 bash[21353]: cephadm 2024-04-13T07:20:03.686380+0000 mgr.y (mgr.14120) 17 : cephadm 1 Saving service mon spec with placement count:5 2024-04-13T07:20:04.930 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:04 smithi089 bash[21353]: audit 2024-04-13T07:20:03.695635+0000 mon.a (mon.0) 59 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:20:04.930 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:04 smithi089 bash[21353]: audit 2024-04-13T07:20:04.007080+0000 mon.a (mon.0) 60 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:20:06.095 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2024-04-13T07:20:06.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:06 smithi089 bash[21353]: audit 2024-04-13T07:20:05.490525+0000 mgr.y (mgr.14120) 18 : audit 0 from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:20:06.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:06 smithi089 bash[21353]: cephadm 2024-04-13T07:20:05.493055+0000 mgr.y (mgr.14120) 19 : cephadm 1 Saving service mgr spec with placement count:2 2024-04-13T07:20:06.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:06 smithi089 bash[21353]: audit 2024-04-13T07:20:05.497722+0000 mon.a (mon.0) 61 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:20:08.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:08 smithi089 bash[21353]: audit 2024-04-13T07:20:07.272249+0000 mon.a (mon.0) 62 : audit 1 from='client.? 172.21.15.89:0/3645985393' entity='client.admin' 2024-04-13T07:20:09.469 INFO:teuthology.orchestra.run.smithi089.stdout:Enabling the dashboard module... 2024-04-13T07:20:09.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:09 smithi089 bash[21353]: cluster 2024-04-13T07:20:08.393972+0000 mgr.y (mgr.14120) 20 : cluster 0 pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:09.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:09 smithi089 bash[21353]: audit 2024-04-13T07:20:08.977766+0000 mon.a (mon.0) 63 : audit 1 from='client.? 172.21.15.89:0/3009918853' entity='client.admin' 2024-04-13T07:20:09.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:09 smithi089 bash[21353]: audit 2024-04-13T07:20:09.037323+0000 mon.a (mon.0) 64 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:20:11.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:10 smithi089 bash[21353]: audit 2024-04-13T07:20:09.726114+0000 mon.a (mon.0) 65 : audit 1 from='mgr.14120 172.21.15.89:0/333878278' entity='mgr.y' 2024-04-13T07:20:11.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:10 smithi089 bash[21353]: cluster 2024-04-13T07:20:10.394361+0000 mgr.y (mgr.14120) 21 : cluster 0 pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:12.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:11 smithi089 bash[21353]: audit 2024-04-13T07:20:10.795766+0000 mon.a (mon.0) 66 : audit 1 from='client.? 172.21.15.89:0/3404602719' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2024-04-13T07:20:12.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:11 smithi089 bash[21606]: ignoring --setuser ceph since I am not root 2024-04-13T07:20:12.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:11 smithi089 bash[21606]: ignoring --setgroup ceph since I am not root 2024-04-13T07:20:12.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:11 smithi089 bash[21606]: debug 2024-04-13T07:20:11.945+0000 7fd136599200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:20:13.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:12 smithi089 bash[21353]: audit 2024-04-13T07:20:11.747653+0000 mon.a (mon.0) 67 : audit 1 from='client.? 172.21.15.89:0/3404602719' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2024-04-13T07:20:13.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:12 smithi089 bash[21353]: cluster 2024-04-13T07:20:11.747712+0000 mon.a (mon.0) 68 : cluster 0 mgrmap e9: y(active, since 23s) 2024-04-13T07:20:13.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:13 smithi089 bash[21353]: audit 2024-04-13T07:20:13.613405+0000 mon.a (mon.0) 69 : audit 0 from='client.? 172.21.15.89:0/3281641859' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2024-04-13T07:20:13.805 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:13 smithi089 bash[21606]: debug 2024-04-13T07:20:13.517+0000 7fd136599200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:20:13.805 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:13 smithi089 bash[21606]: debug 2024-04-13T07:20:13.613+0000 7fd136599200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:20:14.175 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-04-13T07:20:14.175 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "epoch": 9, 2024-04-13T07:20:14.175 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "available": true, 2024-04-13T07:20:14.175 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "active_name": "y", 2024-04-13T07:20:14.175 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "num_standby": 0 2024-04-13T07:20:14.175 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-04-13T07:20:14.175 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for the mgr to restart... 2024-04-13T07:20:14.175 INFO:teuthology.orchestra.run.smithi089.stdout:Waiting for mgr epoch 9... 2024-04-13T07:20:14.191 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:13 smithi089 bash[21606]: debug 2024-04-13T07:20:13.865+0000 7fd136599200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:20:15.054 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:14 smithi089 bash[21606]: debug 2024-04-13T07:20:14.657+0000 7fd136599200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:20:15.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:14 smithi089 bash[21606]: debug 2024-04-13T07:20:14.765+0000 7fd136599200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:20:15.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:14 smithi089 bash[21606]: debug 2024-04-13T07:20:14.869+0000 7fd136599200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:20:15.541 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:15 smithi089 bash[21606]: debug 2024-04-13T07:20:15.229+0000 7fd136599200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:20:15.542 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:15 smithi089 bash[21606]: debug 2024-04-13T07:20:15.345+0000 7fd136599200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:20:15.542 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:15 smithi089 bash[21606]: debug 2024-04-13T07:20:15.449+0000 7fd136599200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:20:16.305 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:15 smithi089 bash[21606]: debug 2024-04-13T07:20:15.809+0000 7fd136599200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:20:16.305 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:16 smithi089 bash[21606]: debug 2024-04-13T07:20:16.049+0000 7fd136599200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:20:16.699 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:16 smithi089 bash[21606]: debug 2024-04-13T07:20:16.309+0000 7fd136599200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:20:16.699 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:16 smithi089 bash[21606]: debug 2024-04-13T07:20:16.420+0000 7fd136599200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:20:17.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:16 smithi089 bash[21606]: debug 2024-04-13T07:20:16.692+0000 7fd136599200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:20:17.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:16 smithi089 bash[21606]: debug 2024-04-13T07:20:16.932+0000 7fd136599200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:20:17.702 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:17 smithi089 bash[21606]: debug 2024-04-13T07:20:17.336+0000 7fd136599200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:20:17.702 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:17 smithi089 bash[21606]: debug 2024-04-13T07:20:17.452+0000 7fd136599200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:20:17.702 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:17 smithi089 bash[21606]: debug 2024-04-13T07:20:17.696+0000 7fd136599200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:20:19.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:18 smithi089 bash[21606]: debug 2024-04-13T07:20:18.688+0000 7fd136599200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:20:19.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:18 smithi089 bash[21606]: debug 2024-04-13T07:20:18.804+0000 7fd136599200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:20:19.555 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21606]: debug 2024-04-13T07:20:19.096+0000 7fd136599200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:20:19.555 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21606]: debug 2024-04-13T07:20:19.208+0000 7fd136599200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:20:19.909 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21606]: debug 2024-04-13T07:20:19.652+0000 7fd136599200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:20:19.909 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21606]: debug 2024-04-13T07:20:19.772+0000 7fd136599200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:20:19.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: cluster 2024-04-13T07:20:19.796690+0000 mon.a (mon.0) 70 : cluster 1 Active manager daemon y restarted 2024-04-13T07:20:19.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: cluster 2024-04-13T07:20:19.796959+0000 mon.a (mon.0) 71 : cluster 1 Activating manager daemon y 2024-04-13T07:20:19.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: cluster 2024-04-13T07:20:19.803155+0000 mon.a (mon.0) 72 : cluster 0 osdmap e3: 0 total, 0 up, 0 in 2024-04-13T07:20:19.909 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: cluster 2024-04-13T07:20:19.803402+0000 mon.a (mon.0) 73 : cluster 0 mgrmap e10: y(active, starting, since 0.00655921s) 2024-04-13T07:20:19.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: audit 2024-04-13T07:20:19.805260+0000 mon.a (mon.0) 74 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:20:19.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: audit 2024-04-13T07:20:19.806651+0000 mon.a (mon.0) 75 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:20:19.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: audit 2024-04-13T07:20:19.807517+0000 mon.a (mon.0) 76 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:20:19.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: audit 2024-04-13T07:20:19.807716+0000 mon.a (mon.0) 77 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:20:19.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: audit 2024-04-13T07:20:19.807883+0000 mon.a (mon.0) 78 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:20:19.910 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:19 smithi089 bash[21353]: cluster 2024-04-13T07:20:19.833226+0000 mon.a (mon.0) 79 : cluster 1 Manager daemon y is now available 2024-04-13T07:20:21.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: audit 2024-04-13T07:20:19.856992+0000 mon.a (mon.0) 80 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:20:21.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: audit 2024-04-13T07:20:19.877261+0000 mon.a (mon.0) 81 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:20:21.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: audit 2024-04-13T07:20:19.880225+0000 mon.a (mon.0) 82 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:20:21.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: cephadm 2024-04-13T07:20:20.092580+0000 mgr.y (mgr.14152) 1 : cephadm 1 [13/Apr/2024:07:20:20] ENGINE Bus STARTING 2024-04-13T07:20:21.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: cephadm 2024-04-13T07:20:20.209716+0000 mgr.y (mgr.14152) 2 : cephadm 1 [13/Apr/2024:07:20:20] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:20:21.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: cephadm 2024-04-13T07:20:20.311175+0000 mgr.y (mgr.14152) 3 : cephadm 1 [13/Apr/2024:07:20:20] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:20:21.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: cephadm 2024-04-13T07:20:20.311342+0000 mgr.y (mgr.14152) 4 : cephadm 1 [13/Apr/2024:07:20:20] ENGINE Bus STARTED 2024-04-13T07:20:21.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: audit 2024-04-13T07:20:20.807524+0000 mgr.y (mgr.14152) 5 : audit 0 from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2024-04-13T07:20:21.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:20 smithi089 bash[21353]: cluster 2024-04-13T07:20:20.810199+0000 mon.a (mon.0) 83 : cluster 0 mgrmap e11: y(active, since 1.01335s) 2024-04-13T07:20:21.477 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout { 2024-04-13T07:20:21.478 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2024-04-13T07:20:21.478 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout "initialized": true 2024-04-13T07:20:21.478 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout } 2024-04-13T07:20:21.478 INFO:teuthology.orchestra.run.smithi089.stdout:mgr epoch 9 is available 2024-04-13T07:20:21.478 INFO:teuthology.orchestra.run.smithi089.stdout:Generating a dashboard self-signed certificate... 2024-04-13T07:20:22.118 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:21 smithi089 bash[21353]: audit 2024-04-13T07:20:20.821253+0000 mgr.y (mgr.14152) 6 : audit 0 from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2024-04-13T07:20:23.133 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:22 smithi089 bash[21353]: cluster 2024-04-13T07:20:21.861570+0000 mon.a (mon.0) 84 : cluster 0 mgrmap e12: y(active, since 2s) 2024-04-13T07:20:23.133 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:22 smithi089 bash[21353]: audit 2024-04-13T07:20:22.677433+0000 mgr.y (mgr.14152) 7 : audit 0 from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:20:23.432 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2024-04-13T07:20:23.432 INFO:teuthology.orchestra.run.smithi089.stdout:Creating initial admin user... 2024-04-13T07:20:24.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:23 smithi089 bash[21353]: audit 2024-04-13T07:20:22.889951+0000 mon.a (mon.0) 85 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:24.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:23 smithi089 bash[21353]: audit 2024-04-13T07:20:22.897294+0000 mon.a (mon.0) 86 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:25.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:24 smithi089 bash[21353]: audit 2024-04-13T07:20:24.688746+0000 mgr.y (mgr.14152) 8 : audit 0 from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:20:25.488 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$jBtMWtoCNp2fKZGlPnrah.fpugftAfuEvtZlq8gFcO3rLE/jE2PhS", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1712992824, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2024-04-13T07:20:25.488 INFO:teuthology.orchestra.run.smithi089.stdout:Fetching dashboard port number... 2024-04-13T07:20:26.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:25 smithi089 bash[21353]: audit 2024-04-13T07:20:24.923401+0000 mon.a (mon.0) 87 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:27.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:26 smithi089 bash[21353]: cluster 2024-04-13T07:20:25.937208+0000 mon.a (mon.0) 88 : cluster 0 mgrmap e13: y(active, since 6s) 2024-04-13T07:20:27.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:26 smithi089 bash[21353]: audit 2024-04-13T07:20:26.667192+0000 mon.a (mon.0) 89 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:27.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:26 smithi089 bash[21353]: audit 2024-04-13T07:20:26.683164+0000 mon.a (mon.0) 90 : audit 0 from='client.? 172.21.15.89:0/2306163381' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2024-04-13T07:20:27.280 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stdout 8443 2024-04-13T07:20:27.280 INFO:teuthology.orchestra.run.smithi089.stdout:firewalld does not appear to be present 2024-04-13T07:20:27.280 INFO:teuthology.orchestra.run.smithi089.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2024-04-13T07:20:27.283 INFO:teuthology.orchestra.run.smithi089.stdout:Ceph Dashboard is now available at: 2024-04-13T07:20:27.283 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:27.283 INFO:teuthology.orchestra.run.smithi089.stdout: URL: https://smithi089.front.sepia.ceph.com:8443/ 2024-04-13T07:20:27.283 INFO:teuthology.orchestra.run.smithi089.stdout: User: admin 2024-04-13T07:20:27.283 INFO:teuthology.orchestra.run.smithi089.stdout: Password: jzlyxpdd62 2024-04-13T07:20:27.283 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:27.284 INFO:teuthology.orchestra.run.smithi089.stdout:Saving cluster configuration to /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config directory 2024-04-13T07:20:27.284 INFO:teuthology.orchestra.run.smithi089.stdout:Enabling autotune for osd_memory_target 2024-04-13T07:20:28.774 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:28 smithi089 bash[21353]: audit 2024-04-13T07:20:27.457008+0000 mon.a (mon.0) 91 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:30.975 INFO:teuthology.orchestra.run.smithi089.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2024-04-13T07:20:30.975 INFO:teuthology.orchestra.run.smithi089.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2024-04-13T07:20:30.975 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:30.975 INFO:teuthology.orchestra.run.smithi089.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2024-04-13T07:20:30.975 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:30.975 INFO:teuthology.orchestra.run.smithi089.stdout:Or, if you are only running a single cluster on this host: 2024-04-13T07:20:30.975 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout:Please consider enabling telemetry to help improve Ceph: 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout: ceph telemetry on 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout:For more information see: 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout: https://docs.ceph.com/en/latest/mgr/telemetry/ 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:20:30.976 INFO:teuthology.orchestra.run.smithi089.stdout:Bootstrap complete. 2024-04-13T07:20:31.021 INFO:tasks.cephadm:Fetching config... 2024-04-13T07:20:31.021 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:20:31.021 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2024-04-13T07:20:31.027 INFO:tasks.cephadm:Fetching client.admin keyring... 2024-04-13T07:20:31.027 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:20:31.028 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2024-04-13T07:20:31.079 INFO:tasks.cephadm:Fetching mon keyring... 2024-04-13T07:20:31.079 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:20:31.080 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/keyring of=/dev/stdout 2024-04-13T07:20:31.132 INFO:tasks.cephadm:Fetching pub ssh key... 2024-04-13T07:20:31.133 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:20:31.133 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2024-04-13T07:20:31.179 INFO:tasks.cephadm:Installing pub ssh key for root users... 2024-04-13T07:20:31.180 DEBUG:teuthology.orchestra.run.smithi089:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWKiuga8eDTQ0XIPnQBkZwFvLzJ75lrKoVzJEqwvE4xNYihT75pT7iZyntoAKDZTYhmQ9Rk99YtD2MienXEPMIrJgvqn+qkTu8cuc0G4LMqhcGOdl3ub7yKG+6lhA72SMZEA/TVPmgs74PMXQ3OWNVCXRvzBEAso5eDr8gYKECGgIHgow013L8LdE+b+9hUg1CQ9Idwsn3jjUm967l558DQny8SsnnaddtSzavHFI2xZh3NLpClOmaRD3XM8FjcW36tGwzrGkMz0QyIH+DnaMqYWBswk7+3GxtoDoBWRLFhMPHNk1FA8c7Z0FNZszCKV+MspxFv3F9+yYKMVtT2MNGhhoLnpeGA7nnp/HuPZGOSK/gDUUvhLD6co4WF437MOmVUe74lx4Dv57u+6fD905cYz9WBDBYjl6etzcbmYhvlukNkiX5tMOR9AEISSwkFY8AqMShyA3yeeXv5SykHO10U8qgMi3lMDcw1bcUPnDab6uel2FG/fa32/n55dce7cU= ceph-ec50a262-f965-11ee-bc8d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-13T07:20:31.244 INFO:teuthology.orchestra.run.smithi089.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWKiuga8eDTQ0XIPnQBkZwFvLzJ75lrKoVzJEqwvE4xNYihT75pT7iZyntoAKDZTYhmQ9Rk99YtD2MienXEPMIrJgvqn+qkTu8cuc0G4LMqhcGOdl3ub7yKG+6lhA72SMZEA/TVPmgs74PMXQ3OWNVCXRvzBEAso5eDr8gYKECGgIHgow013L8LdE+b+9hUg1CQ9Idwsn3jjUm967l558DQny8SsnnaddtSzavHFI2xZh3NLpClOmaRD3XM8FjcW36tGwzrGkMz0QyIH+DnaMqYWBswk7+3GxtoDoBWRLFhMPHNk1FA8c7Z0FNZszCKV+MspxFv3F9+yYKMVtT2MNGhhoLnpeGA7nnp/HuPZGOSK/gDUUvhLD6co4WF437MOmVUe74lx4Dv57u+6fD905cYz9WBDBYjl6etzcbmYhvlukNkiX5tMOR9AEISSwkFY8AqMShyA3yeeXv5SykHO10U8qgMi3lMDcw1bcUPnDab6uel2FG/fa32/n55dce7cU= ceph-ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:20:31.253 DEBUG:teuthology.orchestra.run.smithi203:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWKiuga8eDTQ0XIPnQBkZwFvLzJ75lrKoVzJEqwvE4xNYihT75pT7iZyntoAKDZTYhmQ9Rk99YtD2MienXEPMIrJgvqn+qkTu8cuc0G4LMqhcGOdl3ub7yKG+6lhA72SMZEA/TVPmgs74PMXQ3OWNVCXRvzBEAso5eDr8gYKECGgIHgow013L8LdE+b+9hUg1CQ9Idwsn3jjUm967l558DQny8SsnnaddtSzavHFI2xZh3NLpClOmaRD3XM8FjcW36tGwzrGkMz0QyIH+DnaMqYWBswk7+3GxtoDoBWRLFhMPHNk1FA8c7Z0FNZszCKV+MspxFv3F9+yYKMVtT2MNGhhoLnpeGA7nnp/HuPZGOSK/gDUUvhLD6co4WF437MOmVUe74lx4Dv57u+6fD905cYz9WBDBYjl6etzcbmYhvlukNkiX5tMOR9AEISSwkFY8AqMShyA3yeeXv5SykHO10U8qgMi3lMDcw1bcUPnDab6uel2FG/fa32/n55dce7cU= ceph-ec50a262-f965-11ee-bc8d-c7b262605968' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2024-04-13T07:20:31.276 INFO:teuthology.orchestra.run.smithi203.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDWKiuga8eDTQ0XIPnQBkZwFvLzJ75lrKoVzJEqwvE4xNYihT75pT7iZyntoAKDZTYhmQ9Rk99YtD2MienXEPMIrJgvqn+qkTu8cuc0G4LMqhcGOdl3ub7yKG+6lhA72SMZEA/TVPmgs74PMXQ3OWNVCXRvzBEAso5eDr8gYKECGgIHgow013L8LdE+b+9hUg1CQ9Idwsn3jjUm967l558DQny8SsnnaddtSzavHFI2xZh3NLpClOmaRD3XM8FjcW36tGwzrGkMz0QyIH+DnaMqYWBswk7+3GxtoDoBWRLFhMPHNk1FA8c7Z0FNZszCKV+MspxFv3F9+yYKMVtT2MNGhhoLnpeGA7nnp/HuPZGOSK/gDUUvhLD6co4WF437MOmVUe74lx4Dv57u+6fD905cYz9WBDBYjl6etzcbmYhvlukNkiX5tMOR9AEISSwkFY8AqMShyA3yeeXv5SykHO10U8qgMi3lMDcw1bcUPnDab6uel2FG/fa32/n55dce7cU= ceph-ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:20:31.285 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2024-04-13T07:20:31.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:31 smithi089 bash[21353]: audit 2024-04-13T07:20:30.440130+0000 mon.a (mon.0) 92 : audit 1 from='client.? 172.21.15.89:0/1586170524' entity='client.admin' 2024-04-13T07:20:36.093 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:20:39.499 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.100853+0000 mon.a (mon.0) 93 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:39.499 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.106853+0000 mon.a (mon.0) 94 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:39.499 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.108539+0000 mon.a (mon.0) 95 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi089", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:20:39.499 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.116262+0000 mon.a (mon.0) 96 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:39.499 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.150671+0000 mon.a (mon.0) 97 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:20:39.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.159696+0000 mon.a (mon.0) 98 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:39.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.814262+0000 mon.a (mon.0) 99 : audit 1 from='client.? 172.21.15.89:0/3198416008' entity='client.admin' 2024-04-13T07:20:39.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.858894+0000 mon.a (mon.0) 100 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:20:39.500 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:39 smithi089 bash[21353]: audit 2024-04-13T07:20:38.863421+0000 mon.a (mon.0) 101 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:39.501 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2024-04-13T07:20:39.501 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch client-keyring set client.admin '*' --mode 0755 2024-04-13T07:20:40.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:40 smithi089 bash[21353]: cluster 2024-04-13T07:20:39.809094+0000 mgr.y (mgr.14152) 9 : cluster 0 pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:42.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:41 smithi089 bash[21353]: cluster 2024-04-13T07:20:41.809526+0000 mgr.y (mgr.14152) 10 : cluster 0 pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:44.164 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:20:44.185 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:43 smithi089 bash[21353]: cluster 2024-04-13T07:20:43.809964+0000 mgr.y (mgr.14152) 11 : cluster 0 pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:46.157 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:45 smithi089 bash[21353]: cluster 2024-04-13T07:20:45.810387+0000 mgr.y (mgr.14152) 12 : cluster 0 pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:47.187 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi203 2024-04-13T07:20:47.188 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:20:47.188 DEBUG:teuthology.orchestra.run.smithi203:> dd of=/etc/ceph/ceph.conf 2024-04-13T07:20:47.194 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:20:47.194 DEBUG:teuthology.orchestra.run.smithi203:> dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:20:47.240 INFO:tasks.cephadm:Adding host smithi203 to orchestrator... 2024-04-13T07:20:47.240 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch host add smithi203 2024-04-13T07:20:47.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: audit 2024-04-13T07:20:46.546666+0000 mgr.y (mgr.14152) 13 : audit 0 from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:20:47.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: audit 2024-04-13T07:20:46.551697+0000 mon.a (mon.0) 102 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:47.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: audit 2024-04-13T07:20:46.562769+0000 mon.a (mon.0) 103 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:20:47.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: audit 2024-04-13T07:20:46.564798+0000 mon.a (mon.0) 104 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:20:47.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: audit 2024-04-13T07:20:46.565668+0000 mon.a (mon.0) 105 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:20:47.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: cephadm 2024-04-13T07:20:46.566995+0000 mgr.y (mgr.14152) 14 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:20:47.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: cephadm 2024-04-13T07:20:46.681589+0000 mgr.y (mgr.14152) 15 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:20:47.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: cephadm 2024-04-13T07:20:46.796131+0000 mgr.y (mgr.14152) 16 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:20:47.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: audit 2024-04-13T07:20:47.131167+0000 mon.a (mon.0) 106 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:47.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: audit 2024-04-13T07:20:47.136701+0000 mon.a (mon.0) 107 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:47.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:47 smithi089 bash[21353]: audit 2024-04-13T07:20:47.142355+0000 mon.a (mon.0) 108 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:49.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:48 smithi089 bash[21353]: cephadm 2024-04-13T07:20:46.910439+0000 mgr.y (mgr.14152) 17 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:20:49.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:48 smithi089 bash[21353]: cluster 2024-04-13T07:20:47.810792+0000 mgr.y (mgr.14152) 18 : cluster 0 pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:50.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:49 smithi089 bash[21353]: cluster 2024-04-13T07:20:49.811218+0000 mgr.y (mgr.14152) 19 : cluster 0 pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:51.903 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:20:52.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:51 smithi089 bash[21353]: cluster 2024-04-13T07:20:51.811570+0000 mgr.y (mgr.14152) 20 : cluster 0 pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:54.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:53 smithi089 bash[21353]: cluster 2024-04-13T07:20:53.811993+0000 mgr.y (mgr.14152) 21 : cluster 0 pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:55.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:54 smithi089 bash[21353]: audit 2024-04-13T07:20:54.306421+0000 mgr.y (mgr.14152) 22 : audit 0 from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi203", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:20:56.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:55 smithi089 bash[21353]: cephadm 2024-04-13T07:20:54.995626+0000 mgr.y (mgr.14152) 23 : cephadm 1 Deploying cephadm binary to smithi203 2024-04-13T07:20:56.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:55 smithi089 bash[21353]: cluster 2024-04-13T07:20:55.812403+0000 mgr.y (mgr.14152) 24 : cluster 0 pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:57.594 INFO:teuthology.orchestra.run.smithi089.stdout:Added host 'smithi203' with addr '172.21.15.203' 2024-04-13T07:20:58.239 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch host ls --format=json 2024-04-13T07:20:59.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:58 smithi089 bash[21353]: audit 2024-04-13T07:20:57.592155+0000 mon.a (mon.0) 109 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:20:59.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:58 smithi089 bash[21353]: cephadm 2024-04-13T07:20:57.592943+0000 mgr.y (mgr.14152) 25 : cephadm 1 Added host smithi203 2024-04-13T07:20:59.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:58 smithi089 bash[21353]: audit 2024-04-13T07:20:57.622403+0000 mon.a (mon.0) 110 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:20:59.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:58 smithi089 bash[21353]: cluster 2024-04-13T07:20:57.812771+0000 mgr.y (mgr.14152) 26 : cluster 0 pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:20:59.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:20:58 smithi089 bash[21353]: audit 2024-04-13T07:20:57.946021+0000 mon.a (mon.0) 111 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:00.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:00 smithi089 bash[21353]: audit 2024-04-13T07:20:59.321010+0000 mon.a (mon.0) 112 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:00.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:00 smithi089 bash[21353]: cluster 2024-04-13T07:20:59.813142+0000 mgr.y (mgr.14152) 27 : cluster 0 pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:00.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:00 smithi089 bash[21353]: audit 2024-04-13T07:21:00.222304+0000 mon.a (mon.0) 113 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:02.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:01 smithi089 bash[21353]: cluster 2024-04-13T07:21:01.813570+0000 mgr.y (mgr.14152) 28 : cluster 0 pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:02.907 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:21:04.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:03 smithi089 bash[21353]: cluster 2024-04-13T07:21:03.814017+0000 mgr.y (mgr.14152) 29 : cluster 0 pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:05.330 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:21:05.330 INFO:teuthology.orchestra.run.smithi089.stdout:[{"addr": "172.21.15.89", "hostname": "smithi089", "labels": [], "status": ""}, {"addr": "172.21.15.203", "hostname": "smithi203", "labels": [], "status": ""}] 2024-04-13T07:21:05.851 INFO:tasks.cephadm:Setting crush tunables to default 2024-04-13T07:21:05.852 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd crush tunables default 2024-04-13T07:21:07.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:05.328709+0000 mgr.y (mgr.14152) 30 : audit 0 from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:21:07.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:05.759179+0000 mon.a (mon.0) 114 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:07.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:05.765448+0000 mon.a (mon.0) 115 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:07.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:05.772804+0000 mon.a (mon.0) 116 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:05.778854+0000 mon.a (mon.0) 117 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:05.780418+0000 mon.a (mon.0) 118 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi203", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:05.782621+0000 mon.a (mon.0) 119 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:05.784341+0000 mon.a (mon.0) 120 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: cephadm 2024-04-13T07:21:05.786863+0000 mgr.y (mgr.14152) 31 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: cluster 2024-04-13T07:21:05.814350+0000 mgr.y (mgr.14152) 32 : cluster 0 pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:06.402486+0000 mon.a (mon.0) 121 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:06.409333+0000 mon.a (mon.0) 122 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:07.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:06 smithi089 bash[21353]: audit 2024-04-13T07:21:06.416250+0000 mon.a (mon.0) 123 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:08.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:07 smithi089 bash[21353]: cephadm 2024-04-13T07:21:06.001964+0000 mgr.y (mgr.14152) 33 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:08.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:07 smithi089 bash[21353]: cephadm 2024-04-13T07:21:06.116192+0000 mgr.y (mgr.14152) 34 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:21:08.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:07 smithi089 bash[21353]: cephadm 2024-04-13T07:21:06.231360+0000 mgr.y (mgr.14152) 35 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:21:09.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:08 smithi089 bash[21353]: cluster 2024-04-13T07:21:07.814740+0000 mgr.y (mgr.14152) 36 : cluster 0 pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:10.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:09 smithi089 bash[21353]: cluster 2024-04-13T07:21:09.815151+0000 mgr.y (mgr.14152) 37 : cluster 0 pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:10.516 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:21:12.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:11 smithi089 bash[21353]: cluster 2024-04-13T07:21:11.815527+0000 mgr.y (mgr.14152) 38 : cluster 0 pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:13.150 INFO:teuthology.orchestra.run.smithi089.stderr:adjusted tunables profile to default 2024-04-13T07:21:13.558 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:13 smithi089 bash[21353]: audit 2024-04-13T07:21:13.088602+0000 mon.a (mon.0) 124 : audit 1 from='client.? 172.21.15.89:0/3525481421' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2024-04-13T07:21:13.738 INFO:tasks.cephadm:Adding mon.a on smithi089 2024-04-13T07:21:13.738 INFO:tasks.cephadm:Adding mon.c on smithi089 2024-04-13T07:21:13.738 INFO:tasks.cephadm:Adding mon.b on smithi203 2024-04-13T07:21:13.739 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch apply mon '3;smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi203:172.21.15.203=b' 2024-04-13T07:21:14.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:14 smithi089 bash[21353]: audit 2024-04-13T07:21:13.146059+0000 mon.a (mon.0) 125 : audit 1 from='client.? 172.21.15.89:0/3525481421' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2024-04-13T07:21:14.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:14 smithi089 bash[21353]: cluster 2024-04-13T07:21:13.146158+0000 mon.a (mon.0) 126 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-04-13T07:21:14.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:14 smithi089 bash[21353]: cluster 2024-04-13T07:21:13.815977+0000 mgr.y (mgr.14152) 39 : cluster 0 pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:14.908 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:16.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:15 smithi089 bash[21353]: cluster 2024-04-13T07:21:15.816353+0000 mgr.y (mgr.14152) 40 : cluster 0 pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:17.610 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled mon update... 2024-04-13T07:21:18.467 DEBUG:teuthology.orchestra.run.smithi089:mon.c> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.c.service 2024-04-13T07:21:18.470 DEBUG:teuthology.orchestra.run.smithi203:mon.b> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.b.service 2024-04-13T07:21:18.473 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-13T07:21:18.473 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-13T07:21:18.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: audit 2024-04-13T07:21:17.595842+0000 mgr.y (mgr.14152) 41 : audit 0 from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi203:172.21.15.203=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:21:18.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: cephadm 2024-04-13T07:21:17.601837+0000 mgr.y (mgr.14152) 42 : cephadm 1 Saving service mon spec with placement smithi089:172.21.15.89=a;smithi089:[v2:172.21.15.89:3301,v1:172.21.15.89:6790]=c;smithi203:172.21.15.203=b;count:3 2024-04-13T07:21:18.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: audit 2024-04-13T07:21:17.607889+0000 mon.a (mon.0) 127 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:18.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: audit 2024-04-13T07:21:17.624734+0000 mon.a (mon.0) 128 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:18.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: audit 2024-04-13T07:21:17.628764+0000 mon.a (mon.0) 129 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:18.809 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: audit 2024-04-13T07:21:17.630799+0000 mon.a (mon.0) 130 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:18.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: audit 2024-04-13T07:21:17.639311+0000 mon.a (mon.0) 131 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:18.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: audit 2024-04-13T07:21:17.643331+0000 mon.a (mon.0) 132 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:18.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: audit 2024-04-13T07:21:17.644965+0000 mon.a (mon.0) 133 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:18.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: cephadm 2024-04-13T07:21:17.647098+0000 mgr.y (mgr.14152) 43 : cephadm 1 Deploying daemon mon.b on smithi203 2024-04-13T07:21:18.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:18 smithi089 bash[21353]: cluster 2024-04-13T07:21:17.816737+0000 mgr.y (mgr.14152) 44 : cluster 0 pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:20.145 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:20.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:19 smithi089 bash[21353]: cluster 2024-04-13T07:21:19.817078+0000 mgr.y (mgr.14152) 45 : cluster 0 pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:22.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:21 smithi089 bash[21353]: cluster 2024-04-13T07:21:21.817522+0000 mgr.y (mgr.14152) 46 : cluster 0 pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:23.563 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-04-13T07:21:23.563 INFO:teuthology.orchestra.run.smithi203.stdout:{"epoch":1,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","modified":"2024-04-13T07:19:12.689346Z","created":"2024-04-13T07:19:12.689346Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2024-04-13T07:21:23.565 INFO:teuthology.orchestra.run.smithi203.stderr:dumped monmap epoch 1 2024-04-13T07:21:24.449 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:24 smithi089 bash[21353]: audit 2024-04-13T07:21:23.177154+0000 mon.a (mon.0) 134 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:24.449 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:24 smithi089 bash[21353]: audit 2024-04-13T07:21:23.183577+0000 mon.a (mon.0) 135 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:24.450 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:24 smithi089 bash[21353]: audit 2024-04-13T07:21:23.231044+0000 mon.a (mon.0) 136 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:24.450 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:24 smithi089 bash[21353]: audit 2024-04-13T07:21:23.232196+0000 mon.a (mon.0) 137 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:24.450 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:24 smithi089 bash[21353]: audit 2024-04-13T07:21:23.233951+0000 mon.a (mon.0) 138 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:24.450 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:24 smithi089 bash[21353]: cephadm 2024-04-13T07:21:23.236016+0000 mgr.y (mgr.14152) 47 : cephadm 1 Deploying daemon mon.c on smithi089 2024-04-13T07:21:24.450 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:24 smithi089 bash[21353]: audit 2024-04-13T07:21:23.561374+0000 mon.a (mon.0) 139 : audit 0 from='client.? 172.21.15.203:0/2290454982' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-13T07:21:24.450 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:24 smithi089 bash[21353]: cluster 2024-04-13T07:21:23.817931+0000 mgr.y (mgr.14152) 48 : cluster 0 pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:24.772 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:24 smithi203 bash[23480]: debug 2024-04-13T07:21:24.621+0000 7fae3551a700 1 mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-13T07:21:25.406 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2024-04-13T07:21:25.406 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph mon dump -f json 2024-04-13T07:21:26.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:26.806 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:21:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:27.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:27.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:21:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:28.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:27 smithi089 bash[31522]: debug 2024-04-13T07:21:27.810+0000 7fc976f8c700 1 mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2024-04-13T07:21:29.060 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:24.635535+0000 mon.a (mon.0) 141 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:24.635977+0000 mon.a (mon.0) 142 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:24.636302+0000 mon.a (mon.0) 143 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:25.630628+0000 mon.a (mon.0) 144 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:25.818230+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:26.629892+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:26.632399+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:27.630971+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:27.815323+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:30.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:27.818431+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:28.630738+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:28.816077+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:29.630845+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.680575+0000 mon.a (mon.0) 151 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.691622+0000 mon.a (mon.0) 152 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.711920+0000 mon.a (mon.0) 153 : cluster 0 fsmap 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.712013+0000 mon.a (mon.0) 154 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.712387+0000 mon.a (mon.0) 155 : cluster 0 mgrmap e13: y(active, since 69s) 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.712746+0000 mon.a (mon.0) 156 : cluster 1 overall HEALTH_OK 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:29.720918+0000 mon.a (mon.0) 157 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:30.024 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:29 smithi203 bash[23480]: audit 2024-04-13T07:21:29.728480+0000 mon.a (mon.0) 158 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:30.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:24.635535+0000 mon.a (mon.0) 141 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:21:30.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:24.635977+0000 mon.a (mon.0) 142 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:24.636302+0000 mon.a (mon.0) 143 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:25.630628+0000 mon.a (mon.0) 144 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:25.818230+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:26.629892+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:26.632399+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:27.630971+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:27.815323+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:27.818431+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:28.630738+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:28.816077+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:29.630845+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:30.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.680575+0000 mon.a (mon.0) 151 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-13T07:21:30.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.691622+0000 mon.a (mon.0) 152 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-13T07:21:30.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.711920+0000 mon.a (mon.0) 153 : cluster 0 fsmap 2024-04-13T07:21:30.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.712013+0000 mon.a (mon.0) 154 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-04-13T07:21:30.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.712387+0000 mon.a (mon.0) 155 : cluster 0 mgrmap e13: y(active, since 69s) 2024-04-13T07:21:30.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.712746+0000 mon.a (mon.0) 156 : cluster 1 overall HEALTH_OK 2024-04-13T07:21:30.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:29.720918+0000 mon.a (mon.0) 157 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:30.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:29 smithi089 bash[21353]: audit 2024-04-13T07:21:29.728480+0000 mon.a (mon.0) 158 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:31.056 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:21:30 smithi089 bash[21606]: debug 2024-04-13T07:21:30.630+0000 7fd0f4db9700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-13T07:21:35.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.818754+0000 mgr.y (mgr.14152) 51 : cluster 0 pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:29.822131+0000 mon.a (mon.0) 162 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.822399+0000 mon.a (mon.0) 163 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:29.825251+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:29.840220+0000 mon.a (mon.0) 164 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:29.840451+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:29.888043+0000 mon.a (mon.0) 166 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:30.816517+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:31.816667+0000 mon.a (mon.0) 168 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:31.818660+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:31.819134+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:32.816798+0000 mon.a (mon.0) 169 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:33.816935+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:33.819468+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:34.817180+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:34.827281+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:34.872492+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:34.879605+0000 mon.a (mon.0) 174 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:34.880649+0000 mon.a (mon.0) 175 : cluster 0 fsmap 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:34.880720+0000 mon.a (mon.0) 176 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:34.881000+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e13: y(active, since 75s) 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: cluster 2024-04-13T07:21:34.881261+0000 mon.a (mon.0) 178 : cluster 1 overall HEALTH_OK 2024-04-13T07:21:35.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:34 smithi203 bash[23480]: audit 2024-04-13T07:21:34.909009+0000 mon.a (mon.0) 179 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:35.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.818754+0000 mgr.y (mgr.14152) 51 : cluster 0 pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:29.822131+0000 mon.a (mon.0) 162 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:21:35.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.822399+0000 mon.a (mon.0) 163 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:29.825251+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:29.840220+0000 mon.a (mon.0) 164 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:29.840451+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:29.888043+0000 mon.a (mon.0) 166 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:30.816517+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:31.816667+0000 mon.a (mon.0) 168 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:31.818660+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:31.819134+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:32.816798+0000 mon.a (mon.0) 169 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:33.816935+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:33.819468+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:34.817180+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:34.827281+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:35.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:34.872492+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:21:35.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:34.879605+0000 mon.a (mon.0) 174 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-13T07:21:35.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:34.880649+0000 mon.a (mon.0) 175 : cluster 0 fsmap 2024-04-13T07:21:35.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:34.880720+0000 mon.a (mon.0) 176 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-04-13T07:21:35.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:34.881000+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e13: y(active, since 75s) 2024-04-13T07:21:35.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: cluster 2024-04-13T07:21:34.881261+0000 mon.a (mon.0) 178 : cluster 1 overall HEALTH_OK 2024-04-13T07:21:35.308 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[21353]: audit 2024-04-13T07:21:34.909009+0000 mon.a (mon.0) 179 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:24.635535+0000 mon.a (mon.0) 141 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:24.635977+0000 mon.a (mon.0) 142 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:24.636302+0000 mon.a (mon.0) 143 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:25.630628+0000 mon.a (mon.0) 144 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:25.818230+0000 mgr.y (mgr.14152) 49 : cluster 0 pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:26.629892+0000 mon.a (mon.0) 145 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:26.632399+0000 mon.b (mon.1) 1 : cluster 1 mon.b calling monitor election 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:27.630971+0000 mon.a (mon.0) 146 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:27.815323+0000 mon.a (mon.0) 147 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:27.818431+0000 mgr.y (mgr.14152) 50 : cluster 0 pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:28.630738+0000 mon.a (mon.0) 148 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.309 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:28.816077+0000 mon.a (mon.0) 149 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:29.630845+0000 mon.a (mon.0) 150 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.680575+0000 mon.a (mon.0) 151 : cluster 1 mon.a is new leader, mons a,b in quorum (ranks 0,1) 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.691622+0000 mon.a (mon.0) 152 : cluster 0 monmap e2: 2 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.711920+0000 mon.a (mon.0) 153 : cluster 0 fsmap 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.712013+0000 mon.a (mon.0) 154 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.712387+0000 mon.a (mon.0) 155 : cluster 0 mgrmap e13: y(active, since 69s) 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.712746+0000 mon.a (mon.0) 156 : cluster 1 overall HEALTH_OK 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:29.720918+0000 mon.a (mon.0) 157 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:29.728480+0000 mon.a (mon.0) 158 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.818754+0000 mgr.y (mgr.14152) 51 : cluster 0 pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:29.822131+0000 mon.a (mon.0) 162 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:21:35.310 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.822399+0000 mon.a (mon.0) 163 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:29.825251+0000 mon.b (mon.1) 2 : cluster 1 mon.b calling monitor election 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:29.840220+0000 mon.a (mon.0) 164 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:29.840451+0000 mon.a (mon.0) 165 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:29.888043+0000 mon.a (mon.0) 166 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:30.816517+0000 mon.a (mon.0) 167 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:31.816667+0000 mon.a (mon.0) 168 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:31.818660+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:31.819134+0000 mgr.y (mgr.14152) 52 : cluster 0 pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:32.816798+0000 mon.a (mon.0) 169 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:33.816935+0000 mon.a (mon.0) 170 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:33.819468+0000 mgr.y (mgr.14152) 53 : cluster 0 pgmap v31: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:35.311 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:34.817180+0000 mon.a (mon.0) 171 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:35.312 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:34.827281+0000 mon.a (mon.0) 172 : cluster 1 mon.a calling monitor election 2024-04-13T07:21:35.312 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:34.872492+0000 mon.a (mon.0) 173 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:21:35.312 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:34.879605+0000 mon.a (mon.0) 174 : cluster 0 monmap e3: 3 mons at {a=[v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0],b=[v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0],c=[v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0]} removed_ranks: {} disallowed_leaders: {} 2024-04-13T07:21:35.312 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:34.880649+0000 mon.a (mon.0) 175 : cluster 0 fsmap 2024-04-13T07:21:35.312 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:34.880720+0000 mon.a (mon.0) 176 : cluster 0 osdmap e4: 0 total, 0 up, 0 in 2024-04-13T07:21:35.312 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:34.881000+0000 mon.a (mon.0) 177 : cluster 0 mgrmap e13: y(active, since 75s) 2024-04-13T07:21:35.312 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: cluster 2024-04-13T07:21:34.881261+0000 mon.a (mon.0) 178 : cluster 1 overall HEALTH_OK 2024-04-13T07:21:35.312 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:34 smithi089 bash[31522]: audit 2024-04-13T07:21:34.909009+0000 mon.a (mon.0) 179 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:35.352 INFO:teuthology.orchestra.run.smithi203.stdout: 2024-04-13T07:21:35.353 INFO:teuthology.orchestra.run.smithi203.stdout:{"epoch":3,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","modified":"2024-04-13T07:21:29.816921Z","created":"2024-04-13T07:19:12.689346Z","min_mon_release":18,"min_mon_release_name":"reef","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy","reef"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3300","nonce":0},{"type":"v1","addr":"172.21.15.89:6789","nonce":0}]},"addr":"172.21.15.89:6789/0","public_addr":"172.21.15.89:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:3300","nonce":0},{"type":"v1","addr":"172.21.15.203:6789","nonce":0}]},"addr":"172.21.15.203:6789/0","public_addr":"172.21.15.203:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:3301","nonce":0},{"type":"v1","addr":"172.21.15.89:6790","nonce":0}]},"addr":"172.21.15.89:6790/0","public_addr":"172.21.15.89:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2024-04-13T07:21:35.355 INFO:teuthology.orchestra.run.smithi203.stderr:dumped monmap epoch 3 2024-04-13T07:21:36.082 INFO:tasks.cephadm:Generating final ceph.conf file... 2024-04-13T07:21:36.082 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph config generate-minimal-conf 2024-04-13T07:21:36.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:35 smithi203 bash[23480]: audit 2024-04-13T07:21:35.351008+0000 mon.a (mon.0) 180 : audit 0 from='client.? 172.21.15.203:0/4018769101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-13T07:21:36.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:35 smithi203 bash[23480]: audit 2024-04-13T07:21:35.817186+0000 mon.a (mon.0) 181 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:36.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:35 smithi203 bash[23480]: cluster 2024-04-13T07:21:35.819806+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:36.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:35 smithi089 bash[21353]: audit 2024-04-13T07:21:35.351008+0000 mon.a (mon.0) 180 : audit 0 from='client.? 172.21.15.203:0/4018769101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-13T07:21:36.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:35 smithi089 bash[21353]: audit 2024-04-13T07:21:35.817186+0000 mon.a (mon.0) 181 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:36.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:35 smithi089 bash[21353]: cluster 2024-04-13T07:21:35.819806+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:36.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:35 smithi089 bash[31522]: audit 2024-04-13T07:21:35.351008+0000 mon.a (mon.0) 180 : audit 0 from='client.? 172.21.15.203:0/4018769101' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2024-04-13T07:21:36.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:35 smithi089 bash[31522]: audit 2024-04-13T07:21:35.817186+0000 mon.a (mon.0) 181 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:21:36.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:35 smithi089 bash[31522]: cluster 2024-04-13T07:21:35.819806+0000 mgr.y (mgr.14152) 54 : cluster 0 pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:37.306 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:21:36 smithi089 bash[21606]: debug 2024-04-13T07:21:36.813+0000 7fd0f4db9700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-13T07:21:39.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:38 smithi203 bash[23480]: cluster 2024-04-13T07:21:37.820170+0000 mgr.y (mgr.14152) 55 : cluster 0 pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:39.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:38 smithi089 bash[21353]: cluster 2024-04-13T07:21:37.820170+0000 mgr.y (mgr.14152) 55 : cluster 0 pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:39.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:38 smithi089 bash[31522]: cluster 2024-04-13T07:21:37.820170+0000 mgr.y (mgr.14152) 55 : cluster 0 pgmap v33: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:40.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:39 smithi203 bash[23480]: cluster 2024-04-13T07:21:39.820410+0000 mgr.y (mgr.14152) 56 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:40.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:39 smithi089 bash[21353]: cluster 2024-04-13T07:21:39.820410+0000 mgr.y (mgr.14152) 56 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:40.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:39 smithi089 bash[31522]: cluster 2024-04-13T07:21:39.820410+0000 mgr.y (mgr.14152) 56 : cluster 0 pgmap v34: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:40.744 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:21:41.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:39.968900+0000 mon.a (mon.0) 182 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:39.972814+0000 mon.a (mon.0) 183 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.019552+0000 mon.a (mon.0) 184 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.023125+0000 mon.a (mon.0) 185 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.440344+0000 mon.a (mon.0) 186 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.441888+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: cephadm 2024-04-13T07:21:40.443922+0000 mgr.y (mgr.14152) 57 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: cephadm 2024-04-13T07:21:40.444415+0000 mgr.y (mgr.14152) 58 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: cephadm 2024-04-13T07:21:40.608672+0000 mgr.y (mgr.14152) 59 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: cephadm 2024-04-13T07:21:40.609647+0000 mgr.y (mgr.14152) 60 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.777484+0000 mon.a (mon.0) 188 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.780447+0000 mon.a (mon.0) 189 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.783476+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.786592+0000 mon.a (mon.0) 191 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.793008+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.848632+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.853701+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.860192+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.865337+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.866004+0000 mon.a (mon.0) 197 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.866808+0000 mon.a (mon.0) 198 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[21353]: audit 2024-04-13T07:21:40.867543+0000 mon.a (mon.0) 199 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:39.968900+0000 mon.a (mon.0) 182 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:39.972814+0000 mon.a (mon.0) 183 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.019552+0000 mon.a (mon.0) 184 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.023125+0000 mon.a (mon.0) 185 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.440344+0000 mon.a (mon.0) 186 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.441888+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: cephadm 2024-04-13T07:21:40.443922+0000 mgr.y (mgr.14152) 57 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: cephadm 2024-04-13T07:21:40.444415+0000 mgr.y (mgr.14152) 58 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: cephadm 2024-04-13T07:21:40.608672+0000 mgr.y (mgr.14152) 59 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: cephadm 2024-04-13T07:21:40.609647+0000 mgr.y (mgr.14152) 60 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.777484+0000 mon.a (mon.0) 188 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.780447+0000 mon.a (mon.0) 189 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.783476+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.786592+0000 mon.a (mon.0) 191 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.793008+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.848632+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.853701+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.860192+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.865337+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.866004+0000 mon.a (mon.0) 197 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:41.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.866808+0000 mon.a (mon.0) 198 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:41.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:40 smithi089 bash[31522]: audit 2024-04-13T07:21:40.867543+0000 mon.a (mon.0) 199 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:41.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:39.968900+0000 mon.a (mon.0) 182 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:39.972814+0000 mon.a (mon.0) 183 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.019552+0000 mon.a (mon.0) 184 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.023125+0000 mon.a (mon.0) 185 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.440344+0000 mon.a (mon.0) 186 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.441888+0000 mon.a (mon.0) 187 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: cephadm 2024-04-13T07:21:40.443922+0000 mgr.y (mgr.14152) 57 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: cephadm 2024-04-13T07:21:40.444415+0000 mgr.y (mgr.14152) 58 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: cephadm 2024-04-13T07:21:40.608672+0000 mgr.y (mgr.14152) 59 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: cephadm 2024-04-13T07:21:40.609647+0000 mgr.y (mgr.14152) 60 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.777484+0000 mon.a (mon.0) 188 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.780447+0000 mon.a (mon.0) 189 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.783476+0000 mon.a (mon.0) 190 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.786592+0000 mon.a (mon.0) 191 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.793008+0000 mon.a (mon.0) 192 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.848632+0000 mon.a (mon.0) 193 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.853701+0000 mon.a (mon.0) 194 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.860192+0000 mon.a (mon.0) 195 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.865337+0000 mon.a (mon.0) 196 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:41.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.866004+0000 mon.a (mon.0) 197 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:41.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.866808+0000 mon.a (mon.0) 198 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:41.274 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:40 smithi203 bash[23480]: audit 2024-04-13T07:21:40.867543+0000 mon.a (mon.0) 199 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:42.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:41 smithi203 bash[23480]: cephadm 2024-04-13T07:21:40.865740+0000 mgr.y (mgr.14152) 61 : cephadm 1 Reconfiguring mon.a (unknown last config time)... 2024-04-13T07:21:42.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:41 smithi203 bash[23480]: cephadm 2024-04-13T07:21:40.868451+0000 mgr.y (mgr.14152) 62 : cephadm 1 Reconfiguring daemon mon.a on smithi089 2024-04-13T07:21:42.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:41 smithi203 bash[23480]: cluster 2024-04-13T07:21:41.820741+0000 mgr.y (mgr.14152) 63 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:42.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:41 smithi089 bash[21353]: cephadm 2024-04-13T07:21:40.865740+0000 mgr.y (mgr.14152) 61 : cephadm 1 Reconfiguring mon.a (unknown last config time)... 2024-04-13T07:21:42.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:41 smithi089 bash[21353]: cephadm 2024-04-13T07:21:40.868451+0000 mgr.y (mgr.14152) 62 : cephadm 1 Reconfiguring daemon mon.a on smithi089 2024-04-13T07:21:42.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:41 smithi089 bash[21353]: cluster 2024-04-13T07:21:41.820741+0000 mgr.y (mgr.14152) 63 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:42.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:41 smithi089 bash[31522]: cephadm 2024-04-13T07:21:40.865740+0000 mgr.y (mgr.14152) 61 : cephadm 1 Reconfiguring mon.a (unknown last config time)... 2024-04-13T07:21:42.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:41 smithi089 bash[31522]: cephadm 2024-04-13T07:21:40.868451+0000 mgr.y (mgr.14152) 62 : cephadm 1 Reconfiguring daemon mon.a on smithi089 2024-04-13T07:21:42.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:41 smithi089 bash[31522]: cluster 2024-04-13T07:21:41.820741+0000 mgr.y (mgr.14152) 63 : cluster 0 pgmap v35: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:43.965 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:43 smithi203 bash[23480]: audit 2024-04-13T07:21:42.962116+0000 mon.a (mon.0) 200 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:43.965 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:43 smithi203 bash[23480]: audit 2024-04-13T07:21:42.967546+0000 mon.a (mon.0) 201 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:43.965 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:43 smithi203 bash[23480]: cephadm 2024-04-13T07:21:42.987225+0000 mgr.y (mgr.14152) 64 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-04-13T07:21:43.965 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:43 smithi203 bash[23480]: audit 2024-04-13T07:21:42.987629+0000 mon.a (mon.0) 202 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:44.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[31522]: audit 2024-04-13T07:21:42.962116+0000 mon.a (mon.0) 200 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:44.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[31522]: audit 2024-04-13T07:21:42.967546+0000 mon.a (mon.0) 201 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:44.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[31522]: cephadm 2024-04-13T07:21:42.987225+0000 mgr.y (mgr.14152) 64 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-04-13T07:21:44.238 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[31522]: audit 2024-04-13T07:21:42.987629+0000 mon.a (mon.0) 202 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[31522]: audit 2024-04-13T07:21:42.988448+0000 mon.a (mon.0) 203 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[31522]: audit 2024-04-13T07:21:42.989192+0000 mon.a (mon.0) 204 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[31522]: cephadm 2024-04-13T07:21:42.990094+0000 mgr.y (mgr.14152) 65 : cephadm 1 Reconfiguring daemon mon.c on smithi089 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[31522]: cluster 2024-04-13T07:21:43.821177+0000 mgr.y (mgr.14152) 66 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[21353]: audit 2024-04-13T07:21:42.962116+0000 mon.a (mon.0) 200 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[21353]: audit 2024-04-13T07:21:42.967546+0000 mon.a (mon.0) 201 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[21353]: cephadm 2024-04-13T07:21:42.987225+0000 mgr.y (mgr.14152) 64 : cephadm 1 Reconfiguring mon.c (monmap changed)... 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[21353]: audit 2024-04-13T07:21:42.987629+0000 mon.a (mon.0) 202 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[21353]: audit 2024-04-13T07:21:42.988448+0000 mon.a (mon.0) 203 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:44.239 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[21353]: audit 2024-04-13T07:21:42.989192+0000 mon.a (mon.0) 204 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:44.240 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[21353]: cephadm 2024-04-13T07:21:42.990094+0000 mgr.y (mgr.14152) 65 : cephadm 1 Reconfiguring daemon mon.c on smithi089 2024-04-13T07:21:44.240 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:43 smithi089 bash[21353]: cluster 2024-04-13T07:21:43.821177+0000 mgr.y (mgr.14152) 66 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:44.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:43 smithi203 bash[23480]: audit 2024-04-13T07:21:42.988448+0000 mon.a (mon.0) 203 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:44.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:43 smithi203 bash[23480]: audit 2024-04-13T07:21:42.989192+0000 mon.a (mon.0) 204 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:44.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:43 smithi203 bash[23480]: cephadm 2024-04-13T07:21:42.990094+0000 mgr.y (mgr.14152) 65 : cephadm 1 Reconfiguring daemon mon.c on smithi089 2024-04-13T07:21:44.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:43 smithi203 bash[23480]: cluster 2024-04-13T07:21:43.821177+0000 mgr.y (mgr.14152) 66 : cluster 0 pgmap v36: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:44.447 INFO:teuthology.orchestra.run.smithi089.stdout:# minimal ceph.conf for ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:21:44.447 INFO:teuthology.orchestra.run.smithi089.stdout:[global] 2024-04-13T07:21:44.447 INFO:teuthology.orchestra.run.smithi089.stdout: fsid = ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:21:44.447 INFO:teuthology.orchestra.run.smithi089.stdout: mon_host = [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] 2024-04-13T07:21:45.242 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[21353]: audit 2024-04-13T07:21:44.446010+0000 mon.a (mon.0) 205 : audit 0 from='client.? 172.21.15.89:0/821692695' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:45.242 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[21353]: audit 2024-04-13T07:21:44.921922+0000 mon.a (mon.0) 206 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[21353]: audit 2024-04-13T07:21:44.927279+0000 mon.a (mon.0) 207 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[21353]: audit 2024-04-13T07:21:44.959219+0000 mon.a (mon.0) 208 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[21353]: audit 2024-04-13T07:21:44.960690+0000 mon.a (mon.0) 209 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[21353]: audit 2024-04-13T07:21:44.962117+0000 mon.a (mon.0) 210 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[31522]: audit 2024-04-13T07:21:44.446010+0000 mon.a (mon.0) 205 : audit 0 from='client.? 172.21.15.89:0/821692695' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[31522]: audit 2024-04-13T07:21:44.921922+0000 mon.a (mon.0) 206 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[31522]: audit 2024-04-13T07:21:44.927279+0000 mon.a (mon.0) 207 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[31522]: audit 2024-04-13T07:21:44.959219+0000 mon.a (mon.0) 208 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[31522]: audit 2024-04-13T07:21:44.960690+0000 mon.a (mon.0) 209 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:45.243 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:44 smithi089 bash[31522]: audit 2024-04-13T07:21:44.962117+0000 mon.a (mon.0) 210 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:45.247 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2024-04-13T07:21:45.247 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:21:45.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.conf 2024-04-13T07:21:45.258 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:44 smithi203 bash[23480]: audit 2024-04-13T07:21:44.446010+0000 mon.a (mon.0) 205 : audit 0 from='client.? 172.21.15.89:0/821692695' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:45.259 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:44 smithi203 bash[23480]: audit 2024-04-13T07:21:44.921922+0000 mon.a (mon.0) 206 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:45.259 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:44 smithi203 bash[23480]: audit 2024-04-13T07:21:44.927279+0000 mon.a (mon.0) 207 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:45.259 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:44 smithi203 bash[23480]: audit 2024-04-13T07:21:44.959219+0000 mon.a (mon.0) 208 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:21:45.259 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:44 smithi203 bash[23480]: audit 2024-04-13T07:21:44.960690+0000 mon.a (mon.0) 209 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:21:45.259 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:44 smithi203 bash[23480]: audit 2024-04-13T07:21:44.962117+0000 mon.a (mon.0) 210 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:45.299 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:21:45.300 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:21:45.394 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:21:45.395 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/ceph/ceph.conf 2024-04-13T07:21:45.411 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:21:45.411 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:21:45.469 INFO:tasks.cephadm:Adding mgr.y on smithi089 2024-04-13T07:21:45.470 INFO:tasks.cephadm:Adding mgr.x on smithi203 2024-04-13T07:21:45.470 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch apply mgr '2;smithi089=y;smithi203=x' 2024-04-13T07:21:46.243 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:45 smithi203 bash[23480]: cephadm 2024-04-13T07:21:44.958617+0000 mgr.y (mgr.14152) 67 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-04-13T07:21:46.244 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:45 smithi203 bash[23480]: cephadm 2024-04-13T07:21:44.963670+0000 mgr.y (mgr.14152) 68 : cephadm 1 Reconfiguring daemon mon.b on smithi203 2024-04-13T07:21:46.244 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:45 smithi203 bash[23480]: cluster 2024-04-13T07:21:45.821622+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:46.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:45 smithi089 bash[21353]: cephadm 2024-04-13T07:21:44.958617+0000 mgr.y (mgr.14152) 67 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-04-13T07:21:46.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:45 smithi089 bash[21353]: cephadm 2024-04-13T07:21:44.963670+0000 mgr.y (mgr.14152) 68 : cephadm 1 Reconfiguring daemon mon.b on smithi203 2024-04-13T07:21:46.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:45 smithi089 bash[21353]: cluster 2024-04-13T07:21:45.821622+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:46.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:45 smithi089 bash[31522]: cephadm 2024-04-13T07:21:44.958617+0000 mgr.y (mgr.14152) 67 : cephadm 1 Reconfiguring mon.b (monmap changed)... 2024-04-13T07:21:46.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:45 smithi089 bash[31522]: cephadm 2024-04-13T07:21:44.963670+0000 mgr.y (mgr.14152) 68 : cephadm 1 Reconfiguring daemon mon.b on smithi203 2024-04-13T07:21:46.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:45 smithi089 bash[31522]: cluster 2024-04-13T07:21:45.821622+0000 mgr.y (mgr.14152) 69 : cluster 0 pgmap v37: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:48.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[21353]: audit 2024-04-13T07:21:47.027044+0000 mon.a (mon.0) 211 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[21353]: audit 2024-04-13T07:21:47.034717+0000 mon.a (mon.0) 212 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[21353]: audit 2024-04-13T07:21:47.084072+0000 mon.a (mon.0) 213 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:48.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[21353]: audit 2024-04-13T07:21:47.087855+0000 mon.a (mon.0) 214 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:48.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[21353]: audit 2024-04-13T07:21:47.089663+0000 mon.a (mon.0) 215 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:48.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[21353]: audit 2024-04-13T07:21:47.099294+0000 mon.a (mon.0) 216 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[21353]: cluster 2024-04-13T07:21:47.822054+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:48.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[31522]: audit 2024-04-13T07:21:47.027044+0000 mon.a (mon.0) 211 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[31522]: audit 2024-04-13T07:21:47.034717+0000 mon.a (mon.0) 212 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[31522]: audit 2024-04-13T07:21:47.084072+0000 mon.a (mon.0) 213 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:48.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[31522]: audit 2024-04-13T07:21:47.087855+0000 mon.a (mon.0) 214 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:48.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[31522]: audit 2024-04-13T07:21:47.089663+0000 mon.a (mon.0) 215 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:48.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[31522]: audit 2024-04-13T07:21:47.099294+0000 mon.a (mon.0) 216 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:48 smithi089 bash[31522]: cluster 2024-04-13T07:21:47.822054+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:48.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:48 smithi203 bash[23480]: audit 2024-04-13T07:21:47.027044+0000 mon.a (mon.0) 211 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:48 smithi203 bash[23480]: audit 2024-04-13T07:21:47.034717+0000 mon.a (mon.0) 212 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:48 smithi203 bash[23480]: audit 2024-04-13T07:21:47.084072+0000 mon.a (mon.0) 213 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:48.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:48 smithi203 bash[23480]: audit 2024-04-13T07:21:47.087855+0000 mon.a (mon.0) 214 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:48.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:48 smithi203 bash[23480]: audit 2024-04-13T07:21:47.089663+0000 mon.a (mon.0) 215 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:48.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:48 smithi203 bash[23480]: audit 2024-04-13T07:21:47.099294+0000 mon.a (mon.0) 216 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:48.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:48 smithi203 bash[23480]: cluster 2024-04-13T07:21:47.822054+0000 mgr.y (mgr.14152) 70 : cluster 0 pgmap v38: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:49.168 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:21:50.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:49 smithi203 bash[23480]: cluster 2024-04-13T07:21:49.822451+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:50.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:49 smithi089 bash[21353]: cluster 2024-04-13T07:21:49.822451+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:50.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:49 smithi089 bash[31522]: cluster 2024-04-13T07:21:49.822451+0000 mgr.y (mgr.14152) 71 : cluster 0 pgmap v39: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:51.936 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled mgr update... 2024-04-13T07:21:52.198 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:51 smithi203 bash[23480]: cluster 2024-04-13T07:21:51.822834+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:52.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:51 smithi089 bash[21353]: cluster 2024-04-13T07:21:51.822834+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:52.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:51 smithi089 bash[31522]: cluster 2024-04-13T07:21:51.822834+0000 mgr.y (mgr.14152) 72 : cluster 0 pgmap v40: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:53.072 DEBUG:teuthology.orchestra.run.smithi203:mgr.x> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.x.service 2024-04-13T07:21:53.075 INFO:tasks.cephadm:Deploying OSDs... 2024-04-13T07:21:53.075 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:21:53.075 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/scratch_devs of=/dev/stdout 2024-04-13T07:21:53.082 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-13T07:21:53.082 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_1 2024-04-13T07:21:53.128 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-13T07:21:53.128 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-13T07:21:53.128 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 877 Links: 1 2024-04-13T07:21:53.128 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-13T07:21:53.128 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-04-13 07:12:37.098551179 +0000 2024-04-13T07:21:53.128 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-04-13 07:12:36.846553430 +0000 2024-04-13T07:21:53.128 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-04-13 07:12:36.846553430 +0000 2024-04-13T07:21:53.128 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2024-04-13T07:21:53.128 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-13T07:21:53.183 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-04-13T07:21:53.183 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-04-13T07:21:53.183 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000475778 s, 1.1 MB/s 2024-04-13T07:21:53.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.923796+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi203=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:21:53.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: cephadm 2024-04-13T07:21:51.927432+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi089=y;smithi203=x;count:2 2024-04-13T07:21:53.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.933968+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:53.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.946766+0000 mon.a (mon.0) 218 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:53.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.950410+0000 mon.a (mon.0) 219 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:53.184 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.952180+0000 mon.a (mon.0) 220 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:53.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.961179+0000 mon.a (mon.0) 221 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:53.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.964463+0000 mon.a (mon.0) 222 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:21:53.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.970146+0000 mon.a (mon.0) 223 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-13T07:21:53.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.971929+0000 mon.a (mon.0) 224 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:21:53.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: audit 2024-04-13T07:21:51.973670+0000 mon.a (mon.0) 225 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:53.185 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[31522]: cephadm 2024-04-13T07:21:51.975644+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:21:53.186 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.923796+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi203=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: cephadm 2024-04-13T07:21:51.927432+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi089=y;smithi203=x;count:2 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.933968+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.946766+0000 mon.a (mon.0) 218 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.950410+0000 mon.a (mon.0) 219 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.952180+0000 mon.a (mon.0) 220 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.961179+0000 mon.a (mon.0) 221 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.964463+0000 mon.a (mon.0) 222 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.970146+0000 mon.a (mon.0) 223 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-13T07:21:53.187 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.971929+0000 mon.a (mon.0) 224 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:21:53.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: audit 2024-04-13T07:21:51.973670+0000 mon.a (mon.0) 225 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:53.188 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:52 smithi089 bash[21353]: cephadm 2024-04-13T07:21:51.975644+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:21:53.234 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_2 2024-04-13T07:21:53.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.923796+0000 mgr.y (mgr.14152) 73 : audit 0 from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi089=y;smithi203=x", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:21:53.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: cephadm 2024-04-13T07:21:51.927432+0000 mgr.y (mgr.14152) 74 : cephadm 1 Saving service mgr spec with placement smithi089=y;smithi203=x;count:2 2024-04-13T07:21:53.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.933968+0000 mon.a (mon.0) 217 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:53.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.946766+0000 mon.a (mon.0) 218 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:53.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.950410+0000 mon.a (mon.0) 219 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:53.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.952180+0000 mon.a (mon.0) 220 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:21:53.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.961179+0000 mon.a (mon.0) 221 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:53.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.964463+0000 mon.a (mon.0) 222 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:21:53.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.970146+0000 mon.a (mon.0) 223 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2024-04-13T07:21:53.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.971929+0000 mon.a (mon.0) 224 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:21:53.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: audit 2024-04-13T07:21:51.973670+0000 mon.a (mon.0) 225 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:21:53.273 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:52 smithi203 bash[23480]: cephadm 2024-04-13T07:21:51.975644+0000 mgr.y (mgr.14152) 75 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:21:53.280 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-13T07:21:53.280 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-13T07:21:53.280 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 893 Links: 1 2024-04-13T07:21:53.280 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-13T07:21:53.280 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-04-13 07:12:37.558547070 +0000 2024-04-13T07:21:53.280 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-04-13 07:12:37.310549285 +0000 2024-04-13T07:21:53.280 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-04-13 07:12:37.310549285 +0000 2024-04-13T07:21:53.280 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2024-04-13T07:21:53.280 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-13T07:21:53.334 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-04-13T07:21:53.334 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-04-13T07:21:53.334 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000414769 s, 1.2 MB/s 2024-04-13T07:21:53.336 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-13T07:21:53.385 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_3 2024-04-13T07:21:53.432 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-13T07:21:53.432 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-13T07:21:53.432 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 906 Links: 1 2024-04-13T07:21:53.432 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-13T07:21:53.432 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-04-13 07:12:38.098542247 +0000 2024-04-13T07:21:53.432 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-04-13 07:12:37.814544784 +0000 2024-04-13T07:21:53.432 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-04-13 07:12:37.814544784 +0000 2024-04-13T07:21:53.432 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2024-04-13T07:21:53.433 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-13T07:21:53.486 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-04-13T07:21:53.486 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-04-13T07:21:53.487 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.000419016 s, 1.2 MB/s 2024-04-13T07:21:53.488 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-13T07:21:53.537 DEBUG:teuthology.orchestra.run.smithi089:> stat /dev/vg_nvme/lv_4 2024-04-13T07:21:53.584 INFO:teuthology.orchestra.run.smithi089.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-13T07:21:53.584 INFO:teuthology.orchestra.run.smithi089.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-13T07:21:53.584 INFO:teuthology.orchestra.run.smithi089.stdout:Device: 5h/5d Inode: 925 Links: 1 2024-04-13T07:21:53.584 INFO:teuthology.orchestra.run.smithi089.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-13T07:21:53.584 INFO:teuthology.orchestra.run.smithi089.stdout:Access: 2024-04-13 07:12:38.590537852 +0000 2024-04-13T07:21:53.584 INFO:teuthology.orchestra.run.smithi089.stdout:Modify: 2024-04-13 07:12:38.322540246 +0000 2024-04-13T07:21:53.585 INFO:teuthology.orchestra.run.smithi089.stdout:Change: 2024-04-13 07:12:38.322540246 +0000 2024-04-13T07:21:53.585 INFO:teuthology.orchestra.run.smithi089.stdout: Birth: - 2024-04-13T07:21:53.585 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-13T07:21:53.639 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records in 2024-04-13T07:21:53.639 INFO:teuthology.orchestra.run.smithi089.stderr:1+0 records out 2024-04-13T07:21:53.639 INFO:teuthology.orchestra.run.smithi089.stderr:512 bytes copied, 0.00037636 s, 1.4 MB/s 2024-04-13T07:21:53.640 DEBUG:teuthology.orchestra.run.smithi089:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-13T07:21:53.689 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:21:53.689 DEBUG:teuthology.orchestra.run.smithi203:> dd if=/scratch_devs of=/dev/stdout 2024-04-13T07:21:53.695 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2024-04-13T07:21:53.695 DEBUG:teuthology.orchestra.run.smithi203:> stat /dev/vg_nvme/lv_1 2024-04-13T07:21:53.748 INFO:teuthology.orchestra.run.smithi203.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2024-04-13T07:21:53.748 INFO:teuthology.orchestra.run.smithi203.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-13T07:21:53.748 INFO:teuthology.orchestra.run.smithi203.stdout:Device: 5h/5d Inode: 875 Links: 1 2024-04-13T07:21:53.748 INFO:teuthology.orchestra.run.smithi203.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-13T07:21:53.748 INFO:teuthology.orchestra.run.smithi203.stdout:Access: 2024-04-13 07:12:37.124126923 +0000 2024-04-13T07:21:53.748 INFO:teuthology.orchestra.run.smithi203.stdout:Modify: 2024-04-13 07:12:36.872129173 +0000 2024-04-13T07:21:53.748 INFO:teuthology.orchestra.run.smithi203.stdout:Change: 2024-04-13 07:12:36.872129173 +0000 2024-04-13T07:21:53.748 INFO:teuthology.orchestra.run.smithi203.stdout: Birth: - 2024-04-13T07:21:53.749 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2024-04-13T07:21:53.802 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records in 2024-04-13T07:21:53.802 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records out 2024-04-13T07:21:53.802 INFO:teuthology.orchestra.run.smithi203.stderr:512 bytes copied, 0.000438158 s, 1.2 MB/s 2024-04-13T07:21:53.803 DEBUG:teuthology.orchestra.run.smithi203:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2024-04-13T07:21:53.854 DEBUG:teuthology.orchestra.run.smithi203:> stat /dev/vg_nvme/lv_2 2024-04-13T07:21:53.899 INFO:teuthology.orchestra.run.smithi203.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2024-04-13T07:21:53.899 INFO:teuthology.orchestra.run.smithi203.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-13T07:21:53.900 INFO:teuthology.orchestra.run.smithi203.stdout:Device: 5h/5d Inode: 886 Links: 1 2024-04-13T07:21:53.900 INFO:teuthology.orchestra.run.smithi203.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-13T07:21:53.900 INFO:teuthology.orchestra.run.smithi203.stdout:Access: 2024-04-13 07:12:37.580122850 +0000 2024-04-13T07:21:53.900 INFO:teuthology.orchestra.run.smithi203.stdout:Modify: 2024-04-13 07:12:37.336125029 +0000 2024-04-13T07:21:53.900 INFO:teuthology.orchestra.run.smithi203.stdout:Change: 2024-04-13 07:12:37.336125029 +0000 2024-04-13T07:21:53.900 INFO:teuthology.orchestra.run.smithi203.stdout: Birth: - 2024-04-13T07:21:53.900 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2024-04-13T07:21:53.953 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records in 2024-04-13T07:21:53.953 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records out 2024-04-13T07:21:53.953 INFO:teuthology.orchestra.run.smithi203.stderr:512 bytes copied, 0.000441514 s, 1.2 MB/s 2024-04-13T07:21:53.954 DEBUG:teuthology.orchestra.run.smithi203:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2024-04-13T07:21:54.005 DEBUG:teuthology.orchestra.run.smithi203:> stat /dev/vg_nvme/lv_3 2024-04-13T07:21:54.052 INFO:teuthology.orchestra.run.smithi203.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2024-04-13T07:21:54.052 INFO:teuthology.orchestra.run.smithi203.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-13T07:21:54.052 INFO:teuthology.orchestra.run.smithi203.stdout:Device: 5h/5d Inode: 911 Links: 1 2024-04-13T07:21:54.053 INFO:teuthology.orchestra.run.smithi203.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-13T07:21:54.053 INFO:teuthology.orchestra.run.smithi203.stdout:Access: 2024-04-13 07:12:37.824120669 +0000 2024-04-13T07:21:54.053 INFO:teuthology.orchestra.run.smithi203.stdout:Modify: 2024-04-13 07:12:37.820120706 +0000 2024-04-13T07:21:54.053 INFO:teuthology.orchestra.run.smithi203.stdout:Change: 2024-04-13 07:12:37.820120706 +0000 2024-04-13T07:21:54.053 INFO:teuthology.orchestra.run.smithi203.stdout: Birth: - 2024-04-13T07:21:54.053 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2024-04-13T07:21:54.104 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:53 smithi203 bash[23480]: cluster 2024-04-13T07:21:53.823304+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:54.106 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records in 2024-04-13T07:21:54.106 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records out 2024-04-13T07:21:54.106 INFO:teuthology.orchestra.run.smithi203.stderr:512 bytes copied, 0.00032286 s, 1.6 MB/s 2024-04-13T07:21:54.107 DEBUG:teuthology.orchestra.run.smithi203:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2024-04-13T07:21:54.158 DEBUG:teuthology.orchestra.run.smithi203:> stat /dev/vg_nvme/lv_4 2024-04-13T07:21:54.207 INFO:teuthology.orchestra.run.smithi203.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2024-04-13T07:21:54.207 INFO:teuthology.orchestra.run.smithi203.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2024-04-13T07:21:54.208 INFO:teuthology.orchestra.run.smithi203.stdout:Device: 5h/5d Inode: 926 Links: 1 2024-04-13T07:21:54.208 INFO:teuthology.orchestra.run.smithi203.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2024-04-13T07:21:54.208 INFO:teuthology.orchestra.run.smithi203.stdout:Access: 2024-04-13 07:12:38.300116419 +0000 2024-04-13T07:21:54.208 INFO:teuthology.orchestra.run.smithi203.stdout:Modify: 2024-04-13 07:12:38.296116455 +0000 2024-04-13T07:21:54.208 INFO:teuthology.orchestra.run.smithi203.stdout:Change: 2024-04-13 07:12:38.296116455 +0000 2024-04-13T07:21:54.208 INFO:teuthology.orchestra.run.smithi203.stdout: Birth: - 2024-04-13T07:21:54.208 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2024-04-13T07:21:54.266 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records in 2024-04-13T07:21:54.266 INFO:teuthology.orchestra.run.smithi203.stderr:1+0 records out 2024-04-13T07:21:54.266 INFO:teuthology.orchestra.run.smithi203.stderr:512 bytes copied, 0.00028752 s, 1.8 MB/s 2024-04-13T07:21:54.267 DEBUG:teuthology.orchestra.run.smithi203:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2024-04-13T07:21:54.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:53 smithi089 bash[21353]: cluster 2024-04-13T07:21:53.823304+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:54.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:53 smithi089 bash[31522]: cluster 2024-04-13T07:21:53.823304+0000 mgr.y (mgr.14152) 76 : cluster 0 pgmap v41: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:54.316 INFO:tasks.cephadm:Deploying osd.0 on smithi089 with /dev/vg_nvme/lv_4... 2024-04-13T07:21:54.316 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-13T07:21:54.561 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:54 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:54.561 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:54 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:54.561 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:54 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:54.831 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:54 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:54.831 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:54 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:54.832 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:54 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:21:54.832 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:54 smithi203 systemd[1]: Started Ceph mgr.x for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:21:55.939 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:55 smithi203 bash[24428]: debug 2024-04-13T07:21:55.784+0000 7f289b956700 1 -- 172.21.15.203:0/2134025991 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5599d23701a0 con 0x5599d30f2800 2024-04-13T07:21:55.939 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:55 smithi203 bash[23480]: audit 2024-04-13T07:21:54.825765+0000 mon.a (mon.0) 226 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:55.939 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:55 smithi203 bash[23480]: audit 2024-04-13T07:21:54.833632+0000 mon.a (mon.0) 227 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:55.939 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:55 smithi203 bash[23480]: audit 2024-04-13T07:21:54.853621+0000 mon.a (mon.0) 228 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:55.939 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:55 smithi203 bash[23480]: audit 2024-04-13T07:21:54.861136+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:55.939 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:55 smithi203 bash[23480]: audit 2024-04-13T07:21:54.920052+0000 mon.a (mon.0) 230 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:55.939 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:55 smithi203 bash[23480]: audit 2024-04-13T07:21:54.928993+0000 mon.a (mon.0) 231 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:56.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[21353]: audit 2024-04-13T07:21:54.825765+0000 mon.a (mon.0) 226 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[21353]: audit 2024-04-13T07:21:54.833632+0000 mon.a (mon.0) 227 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[21353]: audit 2024-04-13T07:21:54.853621+0000 mon.a (mon.0) 228 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[21353]: audit 2024-04-13T07:21:54.861136+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[21353]: audit 2024-04-13T07:21:54.920052+0000 mon.a (mon.0) 230 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[21353]: audit 2024-04-13T07:21:54.928993+0000 mon.a (mon.0) 231 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:56.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[31522]: audit 2024-04-13T07:21:54.825765+0000 mon.a (mon.0) 226 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[31522]: audit 2024-04-13T07:21:54.833632+0000 mon.a (mon.0) 227 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[31522]: audit 2024-04-13T07:21:54.853621+0000 mon.a (mon.0) 228 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[31522]: audit 2024-04-13T07:21:54.861136+0000 mon.a (mon.0) 229 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[31522]: audit 2024-04-13T07:21:54.920052+0000 mon.a (mon.0) 230 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:21:56.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:55 smithi089 bash[31522]: audit 2024-04-13T07:21:54.928993+0000 mon.a (mon.0) 231 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:21:57.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:56 smithi203 bash[23480]: cluster 2024-04-13T07:21:55.823796+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:57.272 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:55 smithi203 bash[24428]: debug 2024-04-13T07:21:55.932+0000 7f28a5ada200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:21:57.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:56 smithi089 bash[21353]: cluster 2024-04-13T07:21:55.823796+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:57.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:56 smithi089 bash[31522]: cluster 2024-04-13T07:21:55.823796+0000 mgr.y (mgr.14152) 77 : cluster 0 pgmap v42: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:57.771 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:57 smithi203 bash[24428]: debug 2024-04-13T07:21:57.500+0000 7f28a5ada200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:21:57.772 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:57 smithi203 bash[24428]: debug 2024-04-13T07:21:57.596+0000 7f28a5ada200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:21:58.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:57 smithi203 bash[23480]: cluster 2024-04-13T07:21:57.824278+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:58.272 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:57 smithi203 bash[24428]: debug 2024-04-13T07:21:57.848+0000 7f28a5ada200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:21:58.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:57 smithi089 bash[21353]: cluster 2024-04-13T07:21:57.824278+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:58.355 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:57 smithi089 bash[31522]: cluster 2024-04-13T07:21:57.824278+0000 mgr.y (mgr.14152) 78 : cluster 0 pgmap v43: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:21:58.993 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:21:59.022 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:58 smithi203 bash[24428]: debug 2024-04-13T07:21:58.636+0000 7f28a5ada200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:21:59.022 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:58 smithi203 bash[24428]: debug 2024-04-13T07:21:58.748+0000 7f28a5ada200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:21:59.022 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:58 smithi203 bash[24428]: debug 2024-04-13T07:21:58.848+0000 7f28a5ada200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:21:59.522 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:59 smithi203 bash[24428]: debug 2024-04-13T07:21:59.204+0000 7f28a5ada200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:21:59.522 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:59 smithi203 bash[24428]: debug 2024-04-13T07:21:59.316+0000 7f28a5ada200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:21:59.522 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:59 smithi203 bash[24428]: debug 2024-04-13T07:21:59.416+0000 7f28a5ada200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:22:00.032 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:21:59 smithi203 bash[24428]: debug 2024-04-13T07:21:59.772+0000 7f28a5ada200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:22:00.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:21:59 smithi203 bash[23480]: cluster 2024-04-13T07:21:59.824722+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:00.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:21:59 smithi089 bash[21353]: cluster 2024-04-13T07:21:59.824722+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:00.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:21:59 smithi089 bash[31522]: cluster 2024-04-13T07:21:59.824722+0000 mgr.y (mgr.14152) 79 : cluster 0 pgmap v44: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:00.306 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:00 smithi203 bash[24428]: debug 2024-04-13T07:22:00.028+0000 7f28a5ada200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:22:00.693 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:00 smithi203 bash[24428]: debug 2024-04-13T07:22:00.300+0000 7f28a5ada200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:22:00.693 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:00 smithi203 bash[24428]: debug 2024-04-13T07:22:00.412+0000 7f28a5ada200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:22:01.021 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:00 smithi203 bash[24428]: debug 2024-04-13T07:22:00.688+0000 7f28a5ada200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:22:01.022 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:00 smithi203 bash[24428]: debug 2024-04-13T07:22:00.928+0000 7f28a5ada200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:22:01.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: audit 2024-04-13T07:22:00.018541+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: audit 2024-04-13T07:22:00.026058+0000 mon.a (mon.0) 233 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: audit 2024-04-13T07:22:00.552564+0000 mon.a (mon.0) 234 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:01.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: audit 2024-04-13T07:22:00.554110+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:01.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: audit 2024-04-13T07:22:00.561496+0000 mon.a (mon.0) 236 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: cephadm 2024-04-13T07:22:00.628806+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-04-13T07:22:01.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: audit 2024-04-13T07:22:00.629213+0000 mon.a (mon.0) 237 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:22:01.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: audit 2024-04-13T07:22:00.630276+0000 mon.a (mon.0) 238 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: audit 2024-04-13T07:22:00.631107+0000 mon.a (mon.0) 239 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[21353]: cephadm 2024-04-13T07:22:00.632059+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi089 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: audit 2024-04-13T07:22:00.018541+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: audit 2024-04-13T07:22:00.026058+0000 mon.a (mon.0) 233 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: audit 2024-04-13T07:22:00.552564+0000 mon.a (mon.0) 234 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: audit 2024-04-13T07:22:00.554110+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: audit 2024-04-13T07:22:00.561496+0000 mon.a (mon.0) 236 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: cephadm 2024-04-13T07:22:00.628806+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: audit 2024-04-13T07:22:00.629213+0000 mon.a (mon.0) 237 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: audit 2024-04-13T07:22:00.630276+0000 mon.a (mon.0) 238 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: audit 2024-04-13T07:22:00.631107+0000 mon.a (mon.0) 239 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:01.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:01 smithi089 bash[31522]: cephadm 2024-04-13T07:22:00.632059+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi089 2024-04-13T07:22:01.344 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: audit 2024-04-13T07:22:00.018541+0000 mon.a (mon.0) 232 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.344 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: audit 2024-04-13T07:22:00.026058+0000 mon.a (mon.0) 233 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.344 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: audit 2024-04-13T07:22:00.552564+0000 mon.a (mon.0) 234 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:01.344 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: audit 2024-04-13T07:22:00.554110+0000 mon.a (mon.0) 235 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:01.344 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: audit 2024-04-13T07:22:00.561496+0000 mon.a (mon.0) 236 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:01.344 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: cephadm 2024-04-13T07:22:00.628806+0000 mgr.y (mgr.14152) 80 : cephadm 1 Reconfiguring mgr.y (unknown last config time)... 2024-04-13T07:22:01.344 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: audit 2024-04-13T07:22:00.629213+0000 mon.a (mon.0) 237 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:22:01.345 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: audit 2024-04-13T07:22:00.630276+0000 mon.a (mon.0) 238 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:22:01.345 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: audit 2024-04-13T07:22:00.631107+0000 mon.a (mon.0) 239 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:01.345 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[23480]: cephadm 2024-04-13T07:22:00.632059+0000 mgr.y (mgr.14152) 81 : cephadm 1 Reconfiguring daemon mgr.y on smithi089 2024-04-13T07:22:01.702 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[24428]: debug 2024-04-13T07:22:01.340+0000 7f28a5ada200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:22:01.702 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[24428]: debug 2024-04-13T07:22:01.456+0000 7f28a5ada200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:22:01.702 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:01 smithi203 bash[24428]: debug 2024-04-13T07:22:01.696+0000 7f28a5ada200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:22:02.230 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-13T07:22:02.230 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-13T07:22:02.230 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2024-04-13T07:22:02.230 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2024-04-13T07:22:02.230 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0120646 s, 869 MB/s 2024-04-13T07:22:02.230 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2024-04-13T07:22:02.283 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:02 smithi089 bash[21353]: cluster 2024-04-13T07:22:01.825009+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:02.283 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:02 smithi089 bash[31522]: cluster 2024-04-13T07:22:01.825009+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:02.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:02 smithi203 bash[23480]: cluster 2024-04-13T07:22:01.825009+0000 mgr.y (mgr.14152) 82 : cluster 0 pgmap v45: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:02.947 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi089:vg_nvme/lv_4 2024-04-13T07:22:03.021 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:02 smithi203 bash[24428]: debug 2024-04-13T07:22:02.700+0000 7f28a5ada200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:22:03.022 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:02 smithi203 bash[24428]: debug 2024-04-13T07:22:02.816+0000 7f28a5ada200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:22:03.522 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[24428]: debug 2024-04-13T07:22:03.104+0000 7f28a5ada200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:22:03.522 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[24428]: debug 2024-04-13T07:22:03.216+0000 7f28a5ada200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:22:03.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[21353]: audit 2024-04-13T07:22:02.540810+0000 mon.a (mon.0) 240 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:03.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[21353]: audit 2024-04-13T07:22:02.545684+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:03.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[21353]: audit 2024-04-13T07:22:02.550131+0000 mon.a (mon.0) 242 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:03.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[21353]: audit 2024-04-13T07:22:02.551858+0000 mon.a (mon.0) 243 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:03.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[21353]: audit 2024-04-13T07:22:02.552662+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:03.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[21353]: audit 2024-04-13T07:22:02.559196+0000 mon.a (mon.0) 245 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:03.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[31522]: audit 2024-04-13T07:22:02.540810+0000 mon.a (mon.0) 240 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:03.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[31522]: audit 2024-04-13T07:22:02.545684+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:03.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[31522]: audit 2024-04-13T07:22:02.550131+0000 mon.a (mon.0) 242 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:03.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[31522]: audit 2024-04-13T07:22:02.551858+0000 mon.a (mon.0) 243 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:03.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[31522]: audit 2024-04-13T07:22:02.552662+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:03.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:03 smithi089 bash[31522]: audit 2024-04-13T07:22:02.559196+0000 mon.a (mon.0) 245 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:04.022 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[24428]: debug 2024-04-13T07:22:03.664+0000 7f28a5ada200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:22:04.022 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[24428]: debug 2024-04-13T07:22:03.780+0000 7f28a5ada200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:22:04.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[23480]: audit 2024-04-13T07:22:02.540810+0000 mon.a (mon.0) 240 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:04.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[23480]: audit 2024-04-13T07:22:02.545684+0000 mon.a (mon.0) 241 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:04.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[23480]: audit 2024-04-13T07:22:02.550131+0000 mon.a (mon.0) 242 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:04.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[23480]: audit 2024-04-13T07:22:02.551858+0000 mon.a (mon.0) 243 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:04.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[23480]: audit 2024-04-13T07:22:02.552662+0000 mon.a (mon.0) 244 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:04.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:03 smithi203 bash[23480]: audit 2024-04-13T07:22:02.559196+0000 mon.a (mon.0) 245 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:04.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[21353]: cluster 2024-04-13T07:22:03.805549+0000 mon.a (mon.0) 246 : cluster 0 Standby manager daemon x started 2024-04-13T07:22:04.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[21353]: audit 2024-04-13T07:22:03.807416+0000 mon.b (mon.1) 3 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:22:04.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[21353]: audit 2024-04-13T07:22:03.808536+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:22:04.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[21353]: audit 2024-04-13T07:22:03.809647+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:22:04.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[21353]: audit 2024-04-13T07:22:03.810278+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:22:04.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[21353]: cluster 2024-04-13T07:22:03.825356+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:04.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[31522]: cluster 2024-04-13T07:22:03.805549+0000 mon.a (mon.0) 246 : cluster 0 Standby manager daemon x started 2024-04-13T07:22:04.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[31522]: audit 2024-04-13T07:22:03.807416+0000 mon.b (mon.1) 3 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:22:04.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[31522]: audit 2024-04-13T07:22:03.808536+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:22:04.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[31522]: audit 2024-04-13T07:22:03.809647+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:22:04.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[31522]: audit 2024-04-13T07:22:03.810278+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:22:04.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:04 smithi089 bash[31522]: cluster 2024-04-13T07:22:03.825356+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:05.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:04 smithi203 bash[23480]: cluster 2024-04-13T07:22:03.805549+0000 mon.a (mon.0) 246 : cluster 0 Standby manager daemon x started 2024-04-13T07:22:05.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:04 smithi203 bash[23480]: audit 2024-04-13T07:22:03.807416+0000 mon.b (mon.1) 3 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:22:05.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:04 smithi203 bash[23480]: audit 2024-04-13T07:22:03.808536+0000 mon.b (mon.1) 4 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:22:05.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:04 smithi203 bash[23480]: audit 2024-04-13T07:22:03.809647+0000 mon.b (mon.1) 5 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:22:05.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:04 smithi203 bash[23480]: audit 2024-04-13T07:22:03.810278+0000 mon.b (mon.1) 6 : audit 0 from='mgr.? 172.21.15.203:0/2127983282' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:22:05.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:04 smithi203 bash[23480]: cluster 2024-04-13T07:22:03.825356+0000 mgr.y (mgr.14152) 83 : cluster 0 pgmap v46: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:05.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:05 smithi089 bash[21353]: cluster 2024-04-13T07:22:04.562229+0000 mon.a (mon.0) 247 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-04-13T07:22:05.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:05 smithi089 bash[21353]: audit 2024-04-13T07:22:04.562458+0000 mon.a (mon.0) 248 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:22:05.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:05 smithi089 bash[31522]: cluster 2024-04-13T07:22:04.562229+0000 mon.a (mon.0) 247 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-04-13T07:22:05.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:05 smithi089 bash[31522]: audit 2024-04-13T07:22:04.562458+0000 mon.a (mon.0) 248 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:22:06.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:05 smithi203 bash[23480]: cluster 2024-04-13T07:22:04.562229+0000 mon.a (mon.0) 247 : cluster 0 mgrmap e14: y(active, since 104s), standbys: x 2024-04-13T07:22:06.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:05 smithi203 bash[23480]: audit 2024-04-13T07:22:04.562458+0000 mon.a (mon.0) 248 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:22:07.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:06 smithi203 bash[23480]: cluster 2024-04-13T07:22:05.825723+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:07.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:06 smithi089 bash[21353]: cluster 2024-04-13T07:22:05.825723+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:07.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:06 smithi089 bash[31522]: cluster 2024-04-13T07:22:05.825723+0000 mgr.y (mgr.14152) 84 : cluster 0 pgmap v47: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:07.621 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:22:08.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:07 smithi089 bash[21353]: cluster 2024-04-13T07:22:07.826146+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:08.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:07 smithi089 bash[31522]: cluster 2024-04-13T07:22:07.826146+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:08.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:07 smithi203 bash[23480]: cluster 2024-04-13T07:22:07.826146+0000 mgr.y (mgr.14152) 85 : cluster 0 pgmap v48: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:10.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:09 smithi089 bash[21353]: cluster 2024-04-13T07:22:09.826437+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:10.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:09 smithi089 bash[31522]: cluster 2024-04-13T07:22:09.826437+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:10.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:09 smithi203 bash[23480]: cluster 2024-04-13T07:22:09.826437+0000 mgr.y (mgr.14152) 86 : cluster 0 pgmap v49: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:11.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:10 smithi203 bash[23480]: audit 2024-04-13T07:22:10.063942+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:22:11.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:10 smithi203 bash[23480]: audit 2024-04-13T07:22:10.071268+0000 mon.a (mon.0) 249 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:22:11.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:10 smithi203 bash[23480]: audit 2024-04-13T07:22:10.077758+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:22:11.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:10 smithi203 bash[23480]: audit 2024-04-13T07:22:10.079208+0000 mon.a (mon.0) 251 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:11.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:10 smithi089 bash[21353]: audit 2024-04-13T07:22:10.063942+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:22:11.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:10 smithi089 bash[21353]: audit 2024-04-13T07:22:10.071268+0000 mon.a (mon.0) 249 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:22:11.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:10 smithi089 bash[21353]: audit 2024-04-13T07:22:10.077758+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:22:11.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:10 smithi089 bash[21353]: audit 2024-04-13T07:22:10.079208+0000 mon.a (mon.0) 251 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:11.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:10 smithi089 bash[31522]: audit 2024-04-13T07:22:10.063942+0000 mgr.y (mgr.14152) 87 : audit 0 from='client.24116 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:22:11.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:10 smithi089 bash[31522]: audit 2024-04-13T07:22:10.071268+0000 mon.a (mon.0) 249 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:22:11.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:10 smithi089 bash[31522]: audit 2024-04-13T07:22:10.077758+0000 mon.a (mon.0) 250 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:22:11.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:10 smithi089 bash[31522]: audit 2024-04-13T07:22:10.079208+0000 mon.a (mon.0) 251 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:12.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:11 smithi203 bash[23480]: cluster 2024-04-13T07:22:11.826813+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:12.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:11 smithi089 bash[21353]: cluster 2024-04-13T07:22:11.826813+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:12.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:11 smithi089 bash[31522]: cluster 2024-04-13T07:22:11.826813+0000 mgr.y (mgr.14152) 88 : cluster 0 pgmap v50: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:14.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:13 smithi203 bash[23480]: cluster 2024-04-13T07:22:13.827228+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:14.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:13 smithi089 bash[21353]: cluster 2024-04-13T07:22:13.827228+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:14.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:13 smithi089 bash[31522]: cluster 2024-04-13T07:22:13.827228+0000 mgr.y (mgr.14152) 89 : cluster 0 pgmap v51: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:16.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:15 smithi203 bash[23480]: cluster 2024-04-13T07:22:15.827496+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:16.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:15 smithi089 bash[21353]: cluster 2024-04-13T07:22:15.827496+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:16.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:15 smithi089 bash[31522]: cluster 2024-04-13T07:22:15.827496+0000 mgr.y (mgr.14152) 90 : cluster 0 pgmap v52: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:18.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:17 smithi203 bash[23480]: cluster 2024-04-13T07:22:17.827811+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:18.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:17 smithi203 bash[23480]: audit 2024-04-13T07:22:17.840672+0000 mon.a (mon.0) 252 : audit 1 from='client.? 172.21.15.89:0/3215232316' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9de691f5-25ed-472d-b6dd-dc3908fa85c0"}]: dispatch 2024-04-13T07:22:18.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:17 smithi203 bash[23480]: audit 2024-04-13T07:22:17.849520+0000 mon.a (mon.0) 253 : audit 1 from='client.? 172.21.15.89:0/3215232316' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9de691f5-25ed-472d-b6dd-dc3908fa85c0"}]': finished 2024-04-13T07:22:18.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:17 smithi203 bash[23480]: cluster 2024-04-13T07:22:17.849624+0000 mon.a (mon.0) 254 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-04-13T07:22:18.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:17 smithi203 bash[23480]: audit 2024-04-13T07:22:17.849769+0000 mon.a (mon.0) 255 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:18.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[21353]: cluster 2024-04-13T07:22:17.827811+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:18.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[21353]: audit 2024-04-13T07:22:17.840672+0000 mon.a (mon.0) 252 : audit 1 from='client.? 172.21.15.89:0/3215232316' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9de691f5-25ed-472d-b6dd-dc3908fa85c0"}]: dispatch 2024-04-13T07:22:18.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[21353]: audit 2024-04-13T07:22:17.849520+0000 mon.a (mon.0) 253 : audit 1 from='client.? 172.21.15.89:0/3215232316' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9de691f5-25ed-472d-b6dd-dc3908fa85c0"}]': finished 2024-04-13T07:22:18.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[21353]: cluster 2024-04-13T07:22:17.849624+0000 mon.a (mon.0) 254 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-04-13T07:22:18.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[21353]: audit 2024-04-13T07:22:17.849769+0000 mon.a (mon.0) 255 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:18.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[31522]: cluster 2024-04-13T07:22:17.827811+0000 mgr.y (mgr.14152) 91 : cluster 0 pgmap v53: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:18.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[31522]: audit 2024-04-13T07:22:17.840672+0000 mon.a (mon.0) 252 : audit 1 from='client.? 172.21.15.89:0/3215232316' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9de691f5-25ed-472d-b6dd-dc3908fa85c0"}]: dispatch 2024-04-13T07:22:18.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[31522]: audit 2024-04-13T07:22:17.849520+0000 mon.a (mon.0) 253 : audit 1 from='client.? 172.21.15.89:0/3215232316' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9de691f5-25ed-472d-b6dd-dc3908fa85c0"}]': finished 2024-04-13T07:22:18.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[31522]: cluster 2024-04-13T07:22:17.849624+0000 mon.a (mon.0) 254 : cluster 0 osdmap e5: 1 total, 0 up, 1 in 2024-04-13T07:22:18.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:17 smithi089 bash[31522]: audit 2024-04-13T07:22:17.849769+0000 mon.a (mon.0) 255 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:19.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:18 smithi203 bash[23480]: audit 2024-04-13T07:22:18.562409+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.89:0/150689545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:22:19.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:18 smithi089 bash[21353]: audit 2024-04-13T07:22:18.562409+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.89:0/150689545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:22:19.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:18 smithi089 bash[31522]: audit 2024-04-13T07:22:18.562409+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.89:0/150689545' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:22:20.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:19 smithi203 bash[23480]: cluster 2024-04-13T07:22:19.828324+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:20.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:19 smithi089 bash[21353]: cluster 2024-04-13T07:22:19.828324+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:20.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:19 smithi089 bash[31522]: cluster 2024-04-13T07:22:19.828324+0000 mgr.y (mgr.14152) 92 : cluster 0 pgmap v55: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:22.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:21 smithi203 bash[23480]: cluster 2024-04-13T07:22:21.828712+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:22.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:21 smithi089 bash[21353]: cluster 2024-04-13T07:22:21.828712+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:22.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:21 smithi089 bash[31522]: cluster 2024-04-13T07:22:21.828712+0000 mgr.y (mgr.14152) 93 : cluster 0 pgmap v56: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:24.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:23 smithi203 bash[23480]: cluster 2024-04-13T07:22:23.829194+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:24.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:23 smithi089 bash[21353]: cluster 2024-04-13T07:22:23.829194+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:24.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:23 smithi089 bash[31522]: cluster 2024-04-13T07:22:23.829194+0000 mgr.y (mgr.14152) 94 : cluster 0 pgmap v57: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:26.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:25 smithi203 bash[23480]: cluster 2024-04-13T07:22:25.829631+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:26.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:25 smithi089 bash[21353]: cluster 2024-04-13T07:22:25.829631+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:26.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:25 smithi089 bash[31522]: cluster 2024-04-13T07:22:25.829631+0000 mgr.y (mgr.14152) 95 : cluster 0 pgmap v58: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:28.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:27 smithi203 bash[23480]: cluster 2024-04-13T07:22:27.830040+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:28.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:27 smithi089 bash[21353]: cluster 2024-04-13T07:22:27.830040+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:28.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:27 smithi089 bash[31522]: cluster 2024-04-13T07:22:27.830040+0000 mgr.y (mgr.14152) 96 : cluster 0 pgmap v59: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:30.158 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:29 smithi089 bash[21353]: audit 2024-04-13T07:22:29.815732+0000 mon.a (mon.0) 256 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:22:30.158 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:29 smithi089 bash[21353]: audit 2024-04-13T07:22:29.817433+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:30.158 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:29 smithi089 bash[21353]: cephadm 2024-04-13T07:22:29.818913+0000 mgr.y (mgr.14152) 97 : cephadm 1 Deploying daemon osd.0 on smithi089 2024-04-13T07:22:30.158 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:29 smithi089 bash[21353]: cluster 2024-04-13T07:22:29.830427+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:30.158 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:29 smithi089 bash[31522]: audit 2024-04-13T07:22:29.815732+0000 mon.a (mon.0) 256 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:22:30.158 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:29 smithi089 bash[31522]: audit 2024-04-13T07:22:29.817433+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:30.158 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:29 smithi089 bash[31522]: cephadm 2024-04-13T07:22:29.818913+0000 mgr.y (mgr.14152) 97 : cephadm 1 Deploying daemon osd.0 on smithi089 2024-04-13T07:22:30.159 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:29 smithi089 bash[31522]: cluster 2024-04-13T07:22:29.830427+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:30.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:29 smithi203 bash[23480]: audit 2024-04-13T07:22:29.815732+0000 mon.a (mon.0) 256 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:22:30.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:29 smithi203 bash[23480]: audit 2024-04-13T07:22:29.817433+0000 mon.a (mon.0) 257 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:30.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:29 smithi203 bash[23480]: cephadm 2024-04-13T07:22:29.818913+0000 mgr.y (mgr.14152) 97 : cephadm 1 Deploying daemon osd.0 on smithi089 2024-04-13T07:22:30.272 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:29 smithi203 bash[23480]: cluster 2024-04-13T07:22:29.830427+0000 mgr.y (mgr.14152) 98 : cluster 0 pgmap v60: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:31 smithi089 bash[31522]: cluster 2024-04-13T07:22:31.830859+0000 mgr.y (mgr.14152) 99 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:32.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:31 smithi089 bash[21353]: cluster 2024-04-13T07:22:31.830859+0000 mgr.y (mgr.14152) 99 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:32.275 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:31 smithi203 bash[23480]: cluster 2024-04-13T07:22:31.830859+0000 mgr.y (mgr.14152) 99 : cluster 0 pgmap v61: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:33.760 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:33 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:22:33.761 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:22:33 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:22:33.761 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:33 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:22:34.055 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:22:33 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:22:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:33 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:22:34.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:33 smithi089 bash[21353]: cluster 2024-04-13T07:22:33.831257+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:34.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:33 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:22:34.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:33 smithi089 bash[31522]: cluster 2024-04-13T07:22:33.831257+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:34.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:33 smithi203 bash[23480]: cluster 2024-04-13T07:22:33.831257+0000 mgr.y (mgr.14152) 100 : cluster 0 pgmap v62: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:35.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:34 smithi089 bash[21353]: audit 2024-04-13T07:22:33.900311+0000 mon.a (mon.0) 258 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:35.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:34 smithi089 bash[21353]: audit 2024-04-13T07:22:33.908933+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:35.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:34 smithi089 bash[21353]: audit 2024-04-13T07:22:33.916821+0000 mon.a (mon.0) 260 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:35.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:34 smithi089 bash[31522]: audit 2024-04-13T07:22:33.900311+0000 mon.a (mon.0) 258 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:35.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:34 smithi089 bash[31522]: audit 2024-04-13T07:22:33.908933+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:35.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:34 smithi089 bash[31522]: audit 2024-04-13T07:22:33.916821+0000 mon.a (mon.0) 260 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:35.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:34 smithi203 bash[23480]: audit 2024-04-13T07:22:33.900311+0000 mon.a (mon.0) 258 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:35.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:34 smithi203 bash[23480]: audit 2024-04-13T07:22:33.908933+0000 mon.a (mon.0) 259 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:35.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:34 smithi203 bash[23480]: audit 2024-04-13T07:22:33.916821+0000 mon.a (mon.0) 260 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:36.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:35 smithi203 bash[23480]: cluster 2024-04-13T07:22:35.831699+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:36.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:35 smithi089 bash[21353]: cluster 2024-04-13T07:22:35.831699+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:36.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:35 smithi089 bash[31522]: cluster 2024-04-13T07:22:35.831699+0000 mgr.y (mgr.14152) 101 : cluster 0 pgmap v63: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:39.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:38 smithi089 bash[21353]: cluster 2024-04-13T07:22:37.832196+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:39.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:38 smithi089 bash[31522]: cluster 2024-04-13T07:22:37.832196+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:39.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:38 smithi203 bash[23480]: cluster 2024-04-13T07:22:37.832196+0000 mgr.y (mgr.14152) 102 : cluster 0 pgmap v64: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:40.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:39 smithi089 bash[21353]: audit 2024-04-13T07:22:38.997875+0000 mon.a (mon.0) 261 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:40.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:39 smithi089 bash[21353]: audit 2024-04-13T07:22:39.003586+0000 mon.a (mon.0) 262 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:40.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:39 smithi089 bash[21353]: audit 2024-04-13T07:22:39.004276+0000 mon.a (mon.0) 263 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:40.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:39 smithi089 bash[21353]: audit 2024-04-13T07:22:39.004919+0000 mon.a (mon.0) 264 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:40.259 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:39 smithi089 bash[21353]: audit 2024-04-13T07:22:39.008803+0000 mon.a (mon.0) 265 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:40.259 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:40 smithi089 bash[31522]: audit 2024-04-13T07:22:38.997875+0000 mon.a (mon.0) 261 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:40.259 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:40 smithi089 bash[31522]: audit 2024-04-13T07:22:39.003586+0000 mon.a (mon.0) 262 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:40.260 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:40 smithi089 bash[31522]: audit 2024-04-13T07:22:39.004276+0000 mon.a (mon.0) 263 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:40.260 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:40 smithi089 bash[31522]: audit 2024-04-13T07:22:39.004919+0000 mon.a (mon.0) 264 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:40.260 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:40 smithi089 bash[31522]: audit 2024-04-13T07:22:39.008803+0000 mon.a (mon.0) 265 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:40.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:40 smithi203 bash[23480]: audit 2024-04-13T07:22:38.997875+0000 mon.a (mon.0) 261 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:40.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:40 smithi203 bash[23480]: audit 2024-04-13T07:22:39.003586+0000 mon.a (mon.0) 262 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:40.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:40 smithi203 bash[23480]: audit 2024-04-13T07:22:39.004276+0000 mon.a (mon.0) 263 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:40.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:40 smithi203 bash[23480]: audit 2024-04-13T07:22:39.004919+0000 mon.a (mon.0) 264 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:40.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:40 smithi203 bash[23480]: audit 2024-04-13T07:22:39.008803+0000 mon.a (mon.0) 265 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:41.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:41 smithi203 bash[23480]: cluster 2024-04-13T07:22:39.832621+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:41.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:41 smithi203 bash[23480]: audit 2024-04-13T07:22:40.301925+0000 mon.a (mon.0) 266 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:22:41.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:41 smithi089 bash[21353]: cluster 2024-04-13T07:22:39.832621+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:41.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:41 smithi089 bash[21353]: audit 2024-04-13T07:22:40.301925+0000 mon.a (mon.0) 266 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:22:41.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:41 smithi089 bash[31522]: cluster 2024-04-13T07:22:39.832621+0000 mgr.y (mgr.14152) 103 : cluster 0 pgmap v65: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:41.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:41 smithi089 bash[31522]: audit 2024-04-13T07:22:40.301925+0000 mon.a (mon.0) 266 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:22:42.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:42 smithi089 bash[21353]: audit 2024-04-13T07:22:41.028271+0000 mon.a (mon.0) 267 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-13T07:22:42.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:42 smithi089 bash[21353]: cluster 2024-04-13T07:22:41.028480+0000 mon.a (mon.0) 268 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-04-13T07:22:42.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:42 smithi089 bash[21353]: audit 2024-04-13T07:22:41.028852+0000 mon.a (mon.0) 269 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:42.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:42 smithi089 bash[21353]: audit 2024-04-13T07:22:41.029791+0000 mon.a (mon.0) 270 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:22:42.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:42 smithi089 bash[31522]: audit 2024-04-13T07:22:41.028271+0000 mon.a (mon.0) 267 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-13T07:22:42.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:42 smithi089 bash[31522]: cluster 2024-04-13T07:22:41.028480+0000 mon.a (mon.0) 268 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-04-13T07:22:42.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:42 smithi089 bash[31522]: audit 2024-04-13T07:22:41.028852+0000 mon.a (mon.0) 269 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:42.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:42 smithi089 bash[31522]: audit 2024-04-13T07:22:41.029791+0000 mon.a (mon.0) 270 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:22:42.417 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 0 on host 'smithi089' 2024-04-13T07:22:42.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:42 smithi203 bash[23480]: audit 2024-04-13T07:22:41.028271+0000 mon.a (mon.0) 267 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-13T07:22:42.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:42 smithi203 bash[23480]: cluster 2024-04-13T07:22:41.028480+0000 mon.a (mon.0) 268 : cluster 0 osdmap e6: 1 total, 0 up, 1 in 2024-04-13T07:22:42.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:42 smithi203 bash[23480]: audit 2024-04-13T07:22:41.028852+0000 mon.a (mon.0) 269 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:42.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:42 smithi203 bash[23480]: audit 2024-04-13T07:22:41.029791+0000 mon.a (mon.0) 270 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:22:43.072 DEBUG:teuthology.orchestra.run.smithi089:osd.0> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.0.service 2024-04-13T07:22:43.074 INFO:tasks.cephadm:Deploying osd.1 on smithi089 with /dev/vg_nvme/lv_3... 2024-04-13T07:22:43.074 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-13T07:22:43.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: cluster 2024-04-13T07:22:41.832916+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:43.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: audit 2024-04-13T07:22:42.030271+0000 mon.a (mon.0) 271 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:22:43.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: cluster 2024-04-13T07:22:42.030458+0000 mon.a (mon.0) 272 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-04-13T07:22:43.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: audit 2024-04-13T07:22:42.031267+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: audit 2024-04-13T07:22:42.042624+0000 mon.a (mon.0) 274 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: audit 2024-04-13T07:22:42.331033+0000 mon.a (mon.0) 275 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: audit 2024-04-13T07:22:42.354230+0000 mon.a (mon.0) 276 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: audit 2024-04-13T07:22:42.404678+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[21353]: audit 2024-04-13T07:22:42.513324+0000 mon.a (mon.0) 278 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: cluster 2024-04-13T07:22:41.832916+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: audit 2024-04-13T07:22:42.030271+0000 mon.a (mon.0) 271 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: cluster 2024-04-13T07:22:42.030458+0000 mon.a (mon.0) 272 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: audit 2024-04-13T07:22:42.031267+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: audit 2024-04-13T07:22:42.042624+0000 mon.a (mon.0) 274 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:43.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: audit 2024-04-13T07:22:42.331033+0000 mon.a (mon.0) 275 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:43.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: audit 2024-04-13T07:22:42.354230+0000 mon.a (mon.0) 276 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:43.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: audit 2024-04-13T07:22:42.404678+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:43.307 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:43 smithi089 bash[31522]: audit 2024-04-13T07:22:42.513324+0000 mon.a (mon.0) 278 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' 2024-04-13T07:22:43.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: cluster 2024-04-13T07:22:41.832916+0000 mgr.y (mgr.14152) 104 : cluster 0 pgmap v67: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: audit 2024-04-13T07:22:42.030271+0000 mon.a (mon.0) 271 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:22:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: cluster 2024-04-13T07:22:42.030458+0000 mon.a (mon.0) 272 : cluster 0 osdmap e7: 1 total, 0 up, 1 in 2024-04-13T07:22:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: audit 2024-04-13T07:22:42.031267+0000 mon.a (mon.0) 273 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: audit 2024-04-13T07:22:42.042624+0000 mon.a (mon.0) 274 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: audit 2024-04-13T07:22:42.331033+0000 mon.a (mon.0) 275 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:22:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: audit 2024-04-13T07:22:42.354230+0000 mon.a (mon.0) 276 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: audit 2024-04-13T07:22:42.404678+0000 mon.a (mon.0) 277 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:43 smithi203 bash[23480]: audit 2024-04-13T07:22:42.513324+0000 mon.a (mon.0) 278 : audit 1 from='osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960]' entity='osd.0' 2024-04-13T07:22:44.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[21353]: cluster 2024-04-13T07:22:41.252262+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:22:44.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[21353]: cluster 2024-04-13T07:22:41.252311+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:22:44.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[21353]: audit 2024-04-13T07:22:43.043716+0000 mon.a (mon.0) 279 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:44.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[21353]: cluster 2024-04-13T07:22:43.517743+0000 mon.a (mon.0) 280 : cluster 1 osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960] boot 2024-04-13T07:22:44.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[21353]: cluster 2024-04-13T07:22:43.517850+0000 mon.a (mon.0) 281 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-04-13T07:22:44.305 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[21353]: audit 2024-04-13T07:22:43.518529+0000 mon.a (mon.0) 282 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:44.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[31522]: cluster 2024-04-13T07:22:41.252262+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:22:44.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[31522]: cluster 2024-04-13T07:22:41.252311+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:22:44.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[31522]: audit 2024-04-13T07:22:43.043716+0000 mon.a (mon.0) 279 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:44.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[31522]: cluster 2024-04-13T07:22:43.517743+0000 mon.a (mon.0) 280 : cluster 1 osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960] boot 2024-04-13T07:22:44.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[31522]: cluster 2024-04-13T07:22:43.517850+0000 mon.a (mon.0) 281 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-04-13T07:22:44.306 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:44 smithi089 bash[31522]: audit 2024-04-13T07:22:43.518529+0000 mon.a (mon.0) 282 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:44.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:44 smithi203 bash[23480]: cluster 2024-04-13T07:22:41.252262+0000 osd.0 (osd.0) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:22:44.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:44 smithi203 bash[23480]: cluster 2024-04-13T07:22:41.252311+0000 osd.0 (osd.0) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:22:44.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:44 smithi203 bash[23480]: audit 2024-04-13T07:22:43.043716+0000 mon.a (mon.0) 279 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:44.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:44 smithi203 bash[23480]: cluster 2024-04-13T07:22:43.517743+0000 mon.a (mon.0) 280 : cluster 1 osd.0 [v2:172.21.15.89:6802/3872923960,v1:172.21.15.89:6803/3872923960] boot 2024-04-13T07:22:44.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:44 smithi203 bash[23480]: cluster 2024-04-13T07:22:43.517850+0000 mon.a (mon.0) 281 : cluster 0 osdmap e8: 1 total, 1 up, 1 in 2024-04-13T07:22:44.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:44 smithi203 bash[23480]: audit 2024-04-13T07:22:43.518529+0000 mon.a (mon.0) 282 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:22:45.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:45 smithi203 bash[23480]: cluster 2024-04-13T07:22:43.833278+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:45.772 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:45 smithi203 bash[23480]: cluster 2024-04-13T07:22:45.045066+0000 mon.a (mon.0) 283 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-04-13T07:22:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:45 smithi089 bash[21353]: cluster 2024-04-13T07:22:43.833278+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:45 smithi089 bash[21353]: cluster 2024-04-13T07:22:45.045066+0000 mon.a (mon.0) 283 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-04-13T07:22:45.806 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:45 smithi089 bash[31522]: cluster 2024-04-13T07:22:43.833278+0000 mgr.y (mgr.14152) 105 : cluster 0 pgmap v70: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2024-04-13T07:22:45.807 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:45 smithi089 bash[31522]: cluster 2024-04-13T07:22:45.045066+0000 mon.a (mon.0) 283 : cluster 0 osdmap e9: 1 total, 1 up, 1 in 2024-04-13T07:22:47.767 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:22:47.791 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:47 smithi089 bash[21353]: cluster 2024-04-13T07:22:45.833730+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v72: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:47.791 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:47 smithi089 bash[31522]: cluster 2024-04-13T07:22:45.833730+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v72: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:48.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:47 smithi203 bash[23480]: cluster 2024-04-13T07:22:45.833730+0000 mgr.y (mgr.14152) 106 : cluster 0 pgmap v72: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:49.788 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:49 smithi089 bash[31522]: cluster 2024-04-13T07:22:47.834170+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:49.788 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:49 smithi089 bash[21353]: cluster 2024-04-13T07:22:47.834170+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:50.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:49 smithi203 bash[23480]: cluster 2024-04-13T07:22:47.834170+0000 mgr.y (mgr.14152) 107 : cluster 0 pgmap v73: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:51.518 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-13T07:22:51.519 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-13T07:22:51.519 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2024-04-13T07:22:51.519 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2024-04-13T07:22:51.519 INFO:teuthology.orchestra.run.smithi089.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0157936 s, 664 MB/s 2024-04-13T07:22:51.519 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2024-04-13T07:22:51.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:51 smithi089 bash[21353]: cluster 2024-04-13T07:22:49.834577+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:51.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:51 smithi089 bash[31522]: cluster 2024-04-13T07:22:49.834577+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:52.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:51 smithi203 bash[23480]: cluster 2024-04-13T07:22:49.834577+0000 mgr.y (mgr.14152) 108 : cluster 0 pgmap v74: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:52.327 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi089:vg_nvme/lv_3 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: cluster 2024-04-13T07:22:51.834853+0000 mgr.y (mgr.14152) 109 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: cephadm 2024-04-13T07:22:52.696579+0000 mgr.y (mgr.14152) 110 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: audit 2024-04-13T07:22:52.705213+0000 mon.a (mon.0) 284 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: audit 2024-04-13T07:22:52.714535+0000 mon.a (mon.0) 285 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: audit 2024-04-13T07:22:52.716788+0000 mon.a (mon.0) 286 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: cephadm 2024-04-13T07:22:52.718189+0000 mgr.y (mgr.14152) 111 : cephadm 1 Adjusting osd_memory_target on smithi089 to 16241M 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: audit 2024-04-13T07:22:52.726145+0000 mon.a (mon.0) 287 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: audit 2024-04-13T07:22:52.728351+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: audit 2024-04-13T07:22:52.730019+0000 mon.a (mon.0) 289 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:54.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:53 smithi203 bash[23480]: audit 2024-04-13T07:22:52.739342+0000 mon.a (mon.0) 290 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: cluster 2024-04-13T07:22:51.834853+0000 mgr.y (mgr.14152) 109 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: cephadm 2024-04-13T07:22:52.696579+0000 mgr.y (mgr.14152) 110 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:22:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: audit 2024-04-13T07:22:52.705213+0000 mon.a (mon.0) 284 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: audit 2024-04-13T07:22:52.714535+0000 mon.a (mon.0) 285 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: audit 2024-04-13T07:22:52.716788+0000 mon.a (mon.0) 286 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:22:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: cephadm 2024-04-13T07:22:52.718189+0000 mgr.y (mgr.14152) 111 : cephadm 1 Adjusting osd_memory_target on smithi089 to 16241M 2024-04-13T07:22:54.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: audit 2024-04-13T07:22:52.726145+0000 mon.a (mon.0) 287 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: audit 2024-04-13T07:22:52.728351+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: audit 2024-04-13T07:22:52.730019+0000 mon.a (mon.0) 289 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[21353]: audit 2024-04-13T07:22:52.739342+0000 mon.a (mon.0) 290 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: cluster 2024-04-13T07:22:51.834853+0000 mgr.y (mgr.14152) 109 : cluster 0 pgmap v75: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: cephadm 2024-04-13T07:22:52.696579+0000 mgr.y (mgr.14152) 110 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: audit 2024-04-13T07:22:52.705213+0000 mon.a (mon.0) 284 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: audit 2024-04-13T07:22:52.714535+0000 mon.a (mon.0) 285 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: audit 2024-04-13T07:22:52.716788+0000 mon.a (mon.0) 286 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: cephadm 2024-04-13T07:22:52.718189+0000 mgr.y (mgr.14152) 111 : cephadm 1 Adjusting osd_memory_target on smithi089 to 16241M 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: audit 2024-04-13T07:22:52.726145+0000 mon.a (mon.0) 287 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: audit 2024-04-13T07:22:52.728351+0000 mon.a (mon.0) 288 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:22:54.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: audit 2024-04-13T07:22:52.730019+0000 mon.a (mon.0) 289 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:22:54.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:53 smithi089 bash[31522]: audit 2024-04-13T07:22:52.739342+0000 mon.a (mon.0) 290 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:22:56.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:55 smithi203 bash[23480]: cluster 2024-04-13T07:22:53.835286+0000 mgr.y (mgr.14152) 112 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:56.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:55 smithi089 bash[21353]: cluster 2024-04-13T07:22:53.835286+0000 mgr.y (mgr.14152) 112 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:56.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:55 smithi089 bash[31522]: cluster 2024-04-13T07:22:53.835286+0000 mgr.y (mgr.14152) 112 : cluster 0 pgmap v76: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:57.017 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:22:58.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:57 smithi203 bash[23480]: cluster 2024-04-13T07:22:55.835698+0000 mgr.y (mgr.14152) 113 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:58.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:57 smithi089 bash[21353]: cluster 2024-04-13T07:22:55.835698+0000 mgr.y (mgr.14152) 113 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:22:58.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:57 smithi089 bash[31522]: cluster 2024-04-13T07:22:55.835698+0000 mgr.y (mgr.14152) 113 : cluster 0 pgmap v77: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:00.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:59 smithi203 bash[23480]: cluster 2024-04-13T07:22:57.836027+0000 mgr.y (mgr.14152) 114 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:00.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:59 smithi203 bash[23480]: audit 2024-04-13T07:22:59.666220+0000 mon.a (mon.0) 291 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:23:00.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:59 smithi203 bash[23480]: audit 2024-04-13T07:22:59.669302+0000 mon.a (mon.0) 292 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:23:00.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:22:59 smithi203 bash[23480]: audit 2024-04-13T07:22:59.670236+0000 mon.a (mon.0) 293 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:00.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:59 smithi089 bash[21353]: cluster 2024-04-13T07:22:57.836027+0000 mgr.y (mgr.14152) 114 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:00.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:59 smithi089 bash[21353]: audit 2024-04-13T07:22:59.666220+0000 mon.a (mon.0) 291 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:23:00.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:59 smithi089 bash[21353]: audit 2024-04-13T07:22:59.669302+0000 mon.a (mon.0) 292 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:23:00.055 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:22:59 smithi089 bash[21353]: audit 2024-04-13T07:22:59.670236+0000 mon.a (mon.0) 293 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:00.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:59 smithi089 bash[31522]: cluster 2024-04-13T07:22:57.836027+0000 mgr.y (mgr.14152) 114 : cluster 0 pgmap v78: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:00.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:59 smithi089 bash[31522]: audit 2024-04-13T07:22:59.666220+0000 mon.a (mon.0) 291 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:23:00.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:59 smithi089 bash[31522]: audit 2024-04-13T07:22:59.669302+0000 mon.a (mon.0) 292 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:23:00.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:22:59 smithi089 bash[31522]: audit 2024-04-13T07:22:59.670236+0000 mon.a (mon.0) 293 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:01.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:00 smithi203 bash[23480]: audit 2024-04-13T07:22:59.659575+0000 mgr.y (mgr.14152) 115 : audit 0 from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:23:01.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:00 smithi089 bash[21353]: audit 2024-04-13T07:22:59.659575+0000 mgr.y (mgr.14152) 115 : audit 0 from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:23:01.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:00 smithi089 bash[31522]: audit 2024-04-13T07:22:59.659575+0000 mgr.y (mgr.14152) 115 : audit 0 from='client.24143 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:23:02.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:01 smithi203 bash[23480]: cluster 2024-04-13T07:22:59.836396+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:02.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:01 smithi089 bash[21353]: cluster 2024-04-13T07:22:59.836396+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:01 smithi089 bash[31522]: cluster 2024-04-13T07:22:59.836396+0000 mgr.y (mgr.14152) 116 : cluster 0 pgmap v79: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:04.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:03 smithi203 bash[23480]: cluster 2024-04-13T07:23:01.836870+0000 mgr.y (mgr.14152) 117 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:04.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:03 smithi089 bash[21353]: cluster 2024-04-13T07:23:01.836870+0000 mgr.y (mgr.14152) 117 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:04.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:03 smithi089 bash[31522]: cluster 2024-04-13T07:23:01.836870+0000 mgr.y (mgr.14152) 117 : cluster 0 pgmap v80: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:06.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:05 smithi203 bash[23480]: cluster 2024-04-13T07:23:03.837352+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:06.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:05 smithi089 bash[21353]: cluster 2024-04-13T07:23:03.837352+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:06.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:05 smithi089 bash[31522]: cluster 2024-04-13T07:23:03.837352+0000 mgr.y (mgr.14152) 118 : cluster 0 pgmap v81: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:08.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:07 smithi203 bash[23480]: cluster 2024-04-13T07:23:05.837833+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v82: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:08.069 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:07 smithi203 bash[23480]: audit 2024-04-13T07:23:07.652255+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.89:0/255953545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]: dispatch 2024-04-13T07:23:08.069 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:07 smithi203 bash[23480]: audit 2024-04-13T07:23:07.652752+0000 mon.a (mon.0) 294 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]: dispatch 2024-04-13T07:23:08.070 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:07 smithi203 bash[23480]: audit 2024-04-13T07:23:07.665938+0000 mon.a (mon.0) 295 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]': finished 2024-04-13T07:23:08.070 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:07 smithi203 bash[23480]: cluster 2024-04-13T07:23:07.666117+0000 mon.a (mon.0) 296 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-04-13T07:23:08.070 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:07 smithi203 bash[23480]: audit 2024-04-13T07:23:07.666261+0000 mon.a (mon.0) 297 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:08.070 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[21353]: cluster 2024-04-13T07:23:05.837833+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v82: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:08.070 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[21353]: audit 2024-04-13T07:23:07.652255+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.89:0/255953545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]: dispatch 2024-04-13T07:23:08.070 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[21353]: audit 2024-04-13T07:23:07.652752+0000 mon.a (mon.0) 294 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]: dispatch 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[21353]: audit 2024-04-13T07:23:07.665938+0000 mon.a (mon.0) 295 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]': finished 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[21353]: cluster 2024-04-13T07:23:07.666117+0000 mon.a (mon.0) 296 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[21353]: audit 2024-04-13T07:23:07.666261+0000 mon.a (mon.0) 297 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[31522]: cluster 2024-04-13T07:23:05.837833+0000 mgr.y (mgr.14152) 119 : cluster 0 pgmap v82: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[31522]: audit 2024-04-13T07:23:07.652255+0000 mon.c (mon.2) 3 : audit 1 from='client.? 172.21.15.89:0/255953545' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]: dispatch 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[31522]: audit 2024-04-13T07:23:07.652752+0000 mon.a (mon.0) 294 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]: dispatch 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[31522]: audit 2024-04-13T07:23:07.665938+0000 mon.a (mon.0) 295 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2"}]': finished 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[31522]: cluster 2024-04-13T07:23:07.666117+0000 mon.a (mon.0) 296 : cluster 0 osdmap e10: 2 total, 1 up, 2 in 2024-04-13T07:23:08.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:07 smithi089 bash[31522]: audit 2024-04-13T07:23:07.666261+0000 mon.a (mon.0) 297 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:09.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:08 smithi203 bash[23480]: audit 2024-04-13T07:23:08.438003+0000 mon.a (mon.0) 298 : audit 0 from='client.? 172.21.15.89:0/3910271090' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:23:09.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:08 smithi089 bash[21353]: audit 2024-04-13T07:23:08.438003+0000 mon.a (mon.0) 298 : audit 0 from='client.? 172.21.15.89:0/3910271090' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:23:09.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:08 smithi089 bash[31522]: audit 2024-04-13T07:23:08.438003+0000 mon.a (mon.0) 298 : audit 0 from='client.? 172.21.15.89:0/3910271090' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:23:10.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:09 smithi203 bash[23480]: cluster 2024-04-13T07:23:07.838282+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:10.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:09 smithi089 bash[31522]: cluster 2024-04-13T07:23:07.838282+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:10.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:09 smithi089 bash[21353]: cluster 2024-04-13T07:23:07.838282+0000 mgr.y (mgr.14152) 120 : cluster 0 pgmap v84: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:12.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:11 smithi203 bash[23480]: cluster 2024-04-13T07:23:09.838734+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v85: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:12.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:11 smithi089 bash[21353]: cluster 2024-04-13T07:23:09.838734+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v85: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:12.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:11 smithi089 bash[31522]: cluster 2024-04-13T07:23:09.838734+0000 mgr.y (mgr.14152) 121 : cluster 0 pgmap v85: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:14.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:13 smithi203 bash[23480]: cluster 2024-04-13T07:23:11.839141+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:14.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:13 smithi089 bash[21353]: cluster 2024-04-13T07:23:11.839141+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:14.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:13 smithi089 bash[31522]: cluster 2024-04-13T07:23:11.839141+0000 mgr.y (mgr.14152) 122 : cluster 0 pgmap v86: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:16.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:15 smithi203 bash[23480]: cluster 2024-04-13T07:23:13.839634+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:16.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:15 smithi089 bash[31522]: cluster 2024-04-13T07:23:13.839634+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:16.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:15 smithi089 bash[21353]: cluster 2024-04-13T07:23:13.839634+0000 mgr.y (mgr.14152) 123 : cluster 0 pgmap v87: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:17.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:17 smithi089 bash[21353]: cluster 2024-04-13T07:23:15.840203+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:17.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:17 smithi089 bash[31522]: cluster 2024-04-13T07:23:15.840203+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:18.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:17 smithi203 bash[23480]: cluster 2024-04-13T07:23:15.840203+0000 mgr.y (mgr.14152) 124 : cluster 0 pgmap v88: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:20.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:19 smithi203 bash[23480]: cluster 2024-04-13T07:23:17.840780+0000 mgr.y (mgr.14152) 125 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:20.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:19 smithi089 bash[21353]: cluster 2024-04-13T07:23:17.840780+0000 mgr.y (mgr.14152) 125 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:20.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:19 smithi089 bash[31522]: cluster 2024-04-13T07:23:17.840780+0000 mgr.y (mgr.14152) 125 : cluster 0 pgmap v89: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:21.043 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:20 smithi089 bash[31522]: audit 2024-04-13T07:23:19.847779+0000 mon.a (mon.0) 299 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:23:21.044 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:20 smithi089 bash[31522]: audit 2024-04-13T07:23:19.849266+0000 mon.a (mon.0) 300 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:21.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:20 smithi089 bash[21353]: audit 2024-04-13T07:23:19.847779+0000 mon.a (mon.0) 299 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:23:21.044 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:20 smithi089 bash[21353]: audit 2024-04-13T07:23:19.849266+0000 mon.a (mon.0) 300 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:21.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:20 smithi203 bash[23480]: audit 2024-04-13T07:23:19.847779+0000 mon.a (mon.0) 299 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:23:21.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:20 smithi203 bash[23480]: audit 2024-04-13T07:23:19.849266+0000 mon.a (mon.0) 300 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:22.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:21 smithi089 bash[21353]: cluster 2024-04-13T07:23:19.841212+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:22.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:21 smithi089 bash[21353]: cephadm 2024-04-13T07:23:19.850700+0000 mgr.y (mgr.14152) 127 : cephadm 1 Deploying daemon osd.1 on smithi089 2024-04-13T07:23:22.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:21 smithi089 bash[31522]: cluster 2024-04-13T07:23:19.841212+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:22.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:21 smithi089 bash[31522]: cephadm 2024-04-13T07:23:19.850700+0000 mgr.y (mgr.14152) 127 : cephadm 1 Deploying daemon osd.1 on smithi089 2024-04-13T07:23:22.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:21 smithi203 bash[23480]: cluster 2024-04-13T07:23:19.841212+0000 mgr.y (mgr.14152) 126 : cluster 0 pgmap v90: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:22.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:21 smithi203 bash[23480]: cephadm 2024-04-13T07:23:19.850700+0000 mgr.y (mgr.14152) 127 : cephadm 1 Deploying daemon osd.1 on smithi089 2024-04-13T07:23:23.819 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:23:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:23:23.819 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:23:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:23:23.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:23:23.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:23 smithi089 bash[21353]: cluster 2024-04-13T07:23:21.841747+0000 mgr.y (mgr.14152) 128 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:23.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:23:23.820 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:23 smithi089 bash[31522]: cluster 2024-04-13T07:23:21.841747+0000 mgr.y (mgr.14152) 128 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:24.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:23:24.073 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:23:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:23:24.073 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:23:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:23:24.073 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:23 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:23:24.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:23 smithi203 bash[23480]: cluster 2024-04-13T07:23:21.841747+0000 mgr.y (mgr.14152) 128 : cluster 0 pgmap v91: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:25.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:25 smithi089 bash[21353]: cluster 2024-04-13T07:23:23.842087+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:25.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:25 smithi089 bash[21353]: audit 2024-04-13T07:23:24.070340+0000 mon.a (mon.0) 301 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:25.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:25 smithi089 bash[21353]: audit 2024-04-13T07:23:24.077444+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:25.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:25 smithi089 bash[21353]: audit 2024-04-13T07:23:24.101391+0000 mon.a (mon.0) 303 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:23:25.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:25 smithi089 bash[31522]: cluster 2024-04-13T07:23:23.842087+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:25.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:25 smithi089 bash[31522]: audit 2024-04-13T07:23:24.070340+0000 mon.a (mon.0) 301 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:25.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:25 smithi089 bash[31522]: audit 2024-04-13T07:23:24.077444+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:25.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:25 smithi089 bash[31522]: audit 2024-04-13T07:23:24.101391+0000 mon.a (mon.0) 303 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:23:25.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:25 smithi203 bash[23480]: cluster 2024-04-13T07:23:23.842087+0000 mgr.y (mgr.14152) 129 : cluster 0 pgmap v92: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:25.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:25 smithi203 bash[23480]: audit 2024-04-13T07:23:24.070340+0000 mon.a (mon.0) 301 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:25.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:25 smithi203 bash[23480]: audit 2024-04-13T07:23:24.077444+0000 mon.a (mon.0) 302 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:25.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:25 smithi203 bash[23480]: audit 2024-04-13T07:23:24.101391+0000 mon.a (mon.0) 303 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:23:27.448 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:27 smithi089 bash[31522]: cluster 2024-04-13T07:23:25.842500+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:27.449 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:27 smithi089 bash[21353]: cluster 2024-04-13T07:23:25.842500+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:27.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:27 smithi203 bash[23480]: cluster 2024-04-13T07:23:25.842500+0000 mgr.y (mgr.14152) 130 : cluster 0 pgmap v93: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:29.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:29 smithi089 bash[21353]: cluster 2024-04-13T07:23:27.842963+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v94: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:29.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:29 smithi089 bash[31522]: cluster 2024-04-13T07:23:27.842963+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v94: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:29.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:29 smithi203 bash[23480]: cluster 2024-04-13T07:23:27.842963+0000 mgr.y (mgr.14152) 131 : cluster 0 pgmap v94: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:30.428 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[21353]: audit 2024-04-13T07:23:29.241214+0000 mon.a (mon.0) 304 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:30.428 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[21353]: audit 2024-04-13T07:23:29.247432+0000 mon.a (mon.0) 305 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:30.429 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[21353]: audit 2024-04-13T07:23:29.248212+0000 mon.a (mon.0) 306 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:30.429 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[21353]: audit 2024-04-13T07:23:29.248948+0000 mon.a (mon.0) 307 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:23:30.429 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[21353]: audit 2024-04-13T07:23:29.253311+0000 mon.a (mon.0) 308 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:30.429 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[31522]: audit 2024-04-13T07:23:29.241214+0000 mon.a (mon.0) 304 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:30.429 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[31522]: audit 2024-04-13T07:23:29.247432+0000 mon.a (mon.0) 305 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:30.429 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[31522]: audit 2024-04-13T07:23:29.248212+0000 mon.a (mon.0) 306 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:30.429 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[31522]: audit 2024-04-13T07:23:29.248948+0000 mon.a (mon.0) 307 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:23:30.429 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:30 smithi089 bash[31522]: audit 2024-04-13T07:23:29.253311+0000 mon.a (mon.0) 308 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:30.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:30 smithi203 bash[23480]: audit 2024-04-13T07:23:29.241214+0000 mon.a (mon.0) 304 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:30.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:30 smithi203 bash[23480]: audit 2024-04-13T07:23:29.247432+0000 mon.a (mon.0) 305 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:30.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:30 smithi203 bash[23480]: audit 2024-04-13T07:23:29.248212+0000 mon.a (mon.0) 306 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:30.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:30 smithi203 bash[23480]: audit 2024-04-13T07:23:29.248948+0000 mon.a (mon.0) 307 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:23:30.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:30 smithi203 bash[23480]: audit 2024-04-13T07:23:29.253311+0000 mon.a (mon.0) 308 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:31.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:31 smithi203 bash[23480]: cluster 2024-04-13T07:23:29.843281+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:31.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:31 smithi203 bash[23480]: audit 2024-04-13T07:23:30.433696+0000 mon.c (mon.2) 4 : audit 1 from='osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:23:31.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:31 smithi203 bash[23480]: audit 2024-04-13T07:23:30.434064+0000 mon.a (mon.0) 309 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:23:31.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:31 smithi089 bash[31522]: cluster 2024-04-13T07:23:29.843281+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:31.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:31 smithi089 bash[31522]: audit 2024-04-13T07:23:30.433696+0000 mon.c (mon.2) 4 : audit 1 from='osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:23:31.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:31 smithi089 bash[31522]: audit 2024-04-13T07:23:30.434064+0000 mon.a (mon.0) 309 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:23:31.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:31 smithi089 bash[21353]: cluster 2024-04-13T07:23:29.843281+0000 mgr.y (mgr.14152) 132 : cluster 0 pgmap v95: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:31.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:31 smithi089 bash[21353]: audit 2024-04-13T07:23:30.433696+0000 mon.c (mon.2) 4 : audit 1 from='osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:23:31.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:31 smithi089 bash[21353]: audit 2024-04-13T07:23:30.434064+0000 mon.a (mon.0) 309 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:23:32.432 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 1 on host 'smithi089' 2024-04-13T07:23:32.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[21353]: audit 2024-04-13T07:23:31.271739+0000 mon.a (mon.0) 310 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-13T07:23:32.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[21353]: cluster 2024-04-13T07:23:31.271801+0000 mon.a (mon.0) 311 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-04-13T07:23:32.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[21353]: audit 2024-04-13T07:23:31.271910+0000 mon.a (mon.0) 312 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:32.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[21353]: audit 2024-04-13T07:23:31.272585+0000 mon.c (mon.2) 5 : audit 1 from='osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:23:32.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[21353]: audit 2024-04-13T07:23:31.272992+0000 mon.a (mon.0) 313 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:23:32.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[31522]: audit 2024-04-13T07:23:31.271739+0000 mon.a (mon.0) 310 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-13T07:23:32.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[31522]: cluster 2024-04-13T07:23:31.271801+0000 mon.a (mon.0) 311 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-04-13T07:23:32.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[31522]: audit 2024-04-13T07:23:31.271910+0000 mon.a (mon.0) 312 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:32.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[31522]: audit 2024-04-13T07:23:31.272585+0000 mon.c (mon.2) 5 : audit 1 from='osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:23:32.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:32 smithi089 bash[31522]: audit 2024-04-13T07:23:31.272992+0000 mon.a (mon.0) 313 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:23:32.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:32 smithi203 bash[23480]: audit 2024-04-13T07:23:31.271739+0000 mon.a (mon.0) 310 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-13T07:23:32.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:32 smithi203 bash[23480]: cluster 2024-04-13T07:23:31.271801+0000 mon.a (mon.0) 311 : cluster 0 osdmap e11: 2 total, 1 up, 2 in 2024-04-13T07:23:32.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:32 smithi203 bash[23480]: audit 2024-04-13T07:23:31.271910+0000 mon.a (mon.0) 312 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:32.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:32 smithi203 bash[23480]: audit 2024-04-13T07:23:31.272585+0000 mon.c (mon.2) 5 : audit 1 from='osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:23:32.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:32 smithi203 bash[23480]: audit 2024-04-13T07:23:31.272992+0000 mon.a (mon.0) 313 : audit 1 from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:23:33.128 DEBUG:teuthology.orchestra.run.smithi089:osd.1> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.1.service 2024-04-13T07:23:33.131 INFO:tasks.cephadm:Deploying osd.2 on smithi089 with /dev/vg_nvme/lv_2... 2024-04-13T07:23:33.131 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-13T07:23:33.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[21353]: cluster 2024-04-13T07:23:31.843539+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v97: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:33.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[21353]: audit 2024-04-13T07:23:32.273652+0000 mon.a (mon.0) 314 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[21353]: cluster 2024-04-13T07:23:32.273792+0000 mon.a (mon.0) 315 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[21353]: audit 2024-04-13T07:23:32.274471+0000 mon.a (mon.0) 316 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[21353]: audit 2024-04-13T07:23:32.283207+0000 mon.a (mon.0) 317 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[21353]: audit 2024-04-13T07:23:32.417376+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[21353]: audit 2024-04-13T07:23:32.425493+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[21353]: audit 2024-04-13T07:23:32.441635+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[31522]: cluster 2024-04-13T07:23:31.843539+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v97: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[31522]: audit 2024-04-13T07:23:32.273652+0000 mon.a (mon.0) 314 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[31522]: cluster 2024-04-13T07:23:32.273792+0000 mon.a (mon.0) 315 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[31522]: audit 2024-04-13T07:23:32.274471+0000 mon.a (mon.0) 316 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:33.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[31522]: audit 2024-04-13T07:23:32.283207+0000 mon.a (mon.0) 317 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:33.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[31522]: audit 2024-04-13T07:23:32.417376+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:33.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[31522]: audit 2024-04-13T07:23:32.425493+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:33.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:33 smithi089 bash[31522]: audit 2024-04-13T07:23:32.441635+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:23:33.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:33 smithi203 bash[23480]: cluster 2024-04-13T07:23:31.843539+0000 mgr.y (mgr.14152) 133 : cluster 0 pgmap v97: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:33.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:33 smithi203 bash[23480]: audit 2024-04-13T07:23:32.273652+0000 mon.a (mon.0) 314 : audit 1 from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:23:33.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:33 smithi203 bash[23480]: cluster 2024-04-13T07:23:32.273792+0000 mon.a (mon.0) 315 : cluster 0 osdmap e12: 2 total, 1 up, 2 in 2024-04-13T07:23:33.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:33 smithi203 bash[23480]: audit 2024-04-13T07:23:32.274471+0000 mon.a (mon.0) 316 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:33.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:33 smithi203 bash[23480]: audit 2024-04-13T07:23:32.283207+0000 mon.a (mon.0) 317 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:33.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:33 smithi203 bash[23480]: audit 2024-04-13T07:23:32.417376+0000 mon.a (mon.0) 318 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:33.772 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:33 smithi203 bash[23480]: audit 2024-04-13T07:23:32.425493+0000 mon.a (mon.0) 319 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:33.772 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:33 smithi203 bash[23480]: audit 2024-04-13T07:23:32.441635+0000 mon.a (mon.0) 320 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:23:34.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[21353]: cluster 2024-04-13T07:23:31.382069+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:23:34.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[21353]: cluster 2024-04-13T07:23:31.382188+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:23:34.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[21353]: cluster 2024-04-13T07:23:33.283692+0000 mon.a (mon.0) 321 : cluster 1 osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657] boot 2024-04-13T07:23:34.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[21353]: cluster 2024-04-13T07:23:33.283817+0000 mon.a (mon.0) 322 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-04-13T07:23:34.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[21353]: audit 2024-04-13T07:23:33.284441+0000 mon.a (mon.0) 323 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:34.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[31522]: cluster 2024-04-13T07:23:31.382069+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:23:34.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[31522]: cluster 2024-04-13T07:23:31.382188+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:23:34.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[31522]: cluster 2024-04-13T07:23:33.283692+0000 mon.a (mon.0) 321 : cluster 1 osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657] boot 2024-04-13T07:23:34.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[31522]: cluster 2024-04-13T07:23:33.283817+0000 mon.a (mon.0) 322 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-04-13T07:23:34.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:34 smithi089 bash[31522]: audit 2024-04-13T07:23:33.284441+0000 mon.a (mon.0) 323 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:34.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:34 smithi203 bash[23480]: cluster 2024-04-13T07:23:31.382069+0000 osd.1 (osd.1) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:23:34.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:34 smithi203 bash[23480]: cluster 2024-04-13T07:23:31.382188+0000 osd.1 (osd.1) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:23:34.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:34 smithi203 bash[23480]: cluster 2024-04-13T07:23:33.283692+0000 mon.a (mon.0) 321 : cluster 1 osd.1 [v2:172.21.15.89:6810/302139657,v1:172.21.15.89:6811/302139657] boot 2024-04-13T07:23:34.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:34 smithi203 bash[23480]: cluster 2024-04-13T07:23:33.283817+0000 mon.a (mon.0) 322 : cluster 0 osdmap e13: 2 total, 2 up, 2 in 2024-04-13T07:23:34.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:34 smithi203 bash[23480]: audit 2024-04-13T07:23:33.284441+0000 mon.a (mon.0) 323 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:23:35.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:35 smithi089 bash[21353]: cluster 2024-04-13T07:23:33.844000+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v100: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:35.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:35 smithi089 bash[21353]: cluster 2024-04-13T07:23:34.301195+0000 mon.a (mon.0) 324 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-04-13T07:23:35.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:35 smithi089 bash[31522]: cluster 2024-04-13T07:23:33.844000+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v100: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:35.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:35 smithi089 bash[31522]: cluster 2024-04-13T07:23:34.301195+0000 mon.a (mon.0) 324 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-04-13T07:23:35.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:35 smithi203 bash[23480]: cluster 2024-04-13T07:23:33.844000+0000 mgr.y (mgr.14152) 134 : cluster 0 pgmap v100: 0 pgs: ; 0 B data, 26 MiB used, 89 GiB / 89 GiB avail 2024-04-13T07:23:35.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:35 smithi203 bash[23480]: cluster 2024-04-13T07:23:34.301195+0000 mon.a (mon.0) 324 : cluster 0 osdmap e14: 2 total, 2 up, 2 in 2024-04-13T07:23:37.267 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:23:37.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:37 smithi089 bash[21353]: cluster 2024-04-13T07:23:35.844446+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:37.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:37 smithi089 bash[31522]: cluster 2024-04-13T07:23:35.844446+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:37.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:37 smithi203 bash[23480]: cluster 2024-04-13T07:23:35.844446+0000 mgr.y (mgr.14152) 135 : cluster 0 pgmap v102: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:39.479 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:39 smithi089 bash[31522]: cluster 2024-04-13T07:23:37.844915+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v103: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:39.479 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:39 smithi089 bash[21353]: cluster 2024-04-13T07:23:37.844915+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v103: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:39.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:39 smithi203 bash[23480]: cluster 2024-04-13T07:23:37.844915+0000 mgr.y (mgr.14152) 136 : cluster 0 pgmap v103: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:41.092 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-13T07:23:41.092 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-13T07:23:41.092 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2024-04-13T07:23:41.093 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2024-04-13T07:23:41.093 INFO:teuthology.orchestra.run.smithi089.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0137557 s, 762 MB/s 2024-04-13T07:23:41.093 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2024-04-13T07:23:41.446 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:41 smithi089 bash[21353]: cluster 2024-04-13T07:23:39.845408+0000 mgr.y (mgr.14152) 137 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:41.446 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:41 smithi089 bash[31522]: cluster 2024-04-13T07:23:39.845408+0000 mgr.y (mgr.14152) 137 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:41.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:41 smithi203 bash[23480]: cluster 2024-04-13T07:23:39.845408+0000 mgr.y (mgr.14152) 137 : cluster 0 pgmap v104: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:41.844 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi089:vg_nvme/lv_2 2024-04-13T07:23:43.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: cluster 2024-04-13T07:23:41.845746+0000 mgr.y (mgr.14152) 138 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:43.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: cephadm 2024-04-13T07:23:42.180887+0000 mgr.y (mgr.14152) 139 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:23:43.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: audit 2024-04-13T07:23:42.190152+0000 mon.a (mon.0) 325 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: audit 2024-04-13T07:23:42.202215+0000 mon.a (mon.0) 326 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: audit 2024-04-13T07:23:42.204674+0000 mon.a (mon.0) 327 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:23:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: audit 2024-04-13T07:23:42.206650+0000 mon.a (mon.0) 328 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:23:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: cephadm 2024-04-13T07:23:42.208054+0000 mgr.y (mgr.14152) 140 : cephadm 1 Adjusting osd_memory_target on smithi089 to 8120M 2024-04-13T07:23:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: audit 2024-04-13T07:23:42.218475+0000 mon.a (mon.0) 329 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: audit 2024-04-13T07:23:42.220810+0000 mon.a (mon.0) 330 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: audit 2024-04-13T07:23:42.222692+0000 mon.a (mon.0) 331 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:23:43.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:43 smithi203 bash[23480]: audit 2024-04-13T07:23:42.234278+0000 mon.a (mon.0) 332 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: cluster 2024-04-13T07:23:41.845746+0000 mgr.y (mgr.14152) 138 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:43.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: cephadm 2024-04-13T07:23:42.180887+0000 mgr.y (mgr.14152) 139 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:23:43.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: audit 2024-04-13T07:23:42.190152+0000 mon.a (mon.0) 325 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: audit 2024-04-13T07:23:42.202215+0000 mon.a (mon.0) 326 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: audit 2024-04-13T07:23:42.204674+0000 mon.a (mon.0) 327 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: audit 2024-04-13T07:23:42.206650+0000 mon.a (mon.0) 328 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: cephadm 2024-04-13T07:23:42.208054+0000 mgr.y (mgr.14152) 140 : cephadm 1 Adjusting osd_memory_target on smithi089 to 8120M 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: audit 2024-04-13T07:23:42.218475+0000 mon.a (mon.0) 329 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: audit 2024-04-13T07:23:42.220810+0000 mon.a (mon.0) 330 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: audit 2024-04-13T07:23:42.222692+0000 mon.a (mon.0) 331 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[21353]: audit 2024-04-13T07:23:42.234278+0000 mon.a (mon.0) 332 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: cluster 2024-04-13T07:23:41.845746+0000 mgr.y (mgr.14152) 138 : cluster 0 pgmap v105: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: cephadm 2024-04-13T07:23:42.180887+0000 mgr.y (mgr.14152) 139 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:23:43.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: audit 2024-04-13T07:23:42.190152+0000 mon.a (mon.0) 325 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: audit 2024-04-13T07:23:42.202215+0000 mon.a (mon.0) 326 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: audit 2024-04-13T07:23:42.204674+0000 mon.a (mon.0) 327 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:23:43.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: audit 2024-04-13T07:23:42.206650+0000 mon.a (mon.0) 328 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:23:43.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: cephadm 2024-04-13T07:23:42.208054+0000 mgr.y (mgr.14152) 140 : cephadm 1 Adjusting osd_memory_target on smithi089 to 8120M 2024-04-13T07:23:43.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: audit 2024-04-13T07:23:42.218475+0000 mon.a (mon.0) 329 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:43.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: audit 2024-04-13T07:23:42.220810+0000 mon.a (mon.0) 330 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:43.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: audit 2024-04-13T07:23:42.222692+0000 mon.a (mon.0) 331 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:23:43.556 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:43 smithi089 bash[31522]: audit 2024-04-13T07:23:42.234278+0000 mon.a (mon.0) 332 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:23:45.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:45 smithi203 bash[23480]: cluster 2024-04-13T07:23:43.846163+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:45.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:45 smithi089 bash[21353]: cluster 2024-04-13T07:23:43.846163+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:45.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:45 smithi089 bash[31522]: cluster 2024-04-13T07:23:43.846163+0000 mgr.y (mgr.14152) 141 : cluster 0 pgmap v106: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:46.541 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:23:47.463 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:47 smithi089 bash[31522]: cluster 2024-04-13T07:23:45.846657+0000 mgr.y (mgr.14152) 142 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:47.463 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:47 smithi089 bash[21353]: cluster 2024-04-13T07:23:45.846657+0000 mgr.y (mgr.14152) 142 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:47.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:47 smithi203 bash[23480]: cluster 2024-04-13T07:23:45.846657+0000 mgr.y (mgr.14152) 142 : cluster 0 pgmap v107: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:49.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:49 smithi203 bash[23480]: cluster 2024-04-13T07:23:47.847157+0000 mgr.y (mgr.14152) 143 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:49.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:49 smithi203 bash[23480]: audit 2024-04-13T07:23:49.063867+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:23:49.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:49 smithi203 bash[23480]: audit 2024-04-13T07:23:49.066771+0000 mon.a (mon.0) 334 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:23:49.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:49 smithi203 bash[23480]: audit 2024-04-13T07:23:49.067747+0000 mon.a (mon.0) 335 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:49.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:49 smithi089 bash[21353]: cluster 2024-04-13T07:23:47.847157+0000 mgr.y (mgr.14152) 143 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:49.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:49 smithi089 bash[21353]: audit 2024-04-13T07:23:49.063867+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:23:49.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:49 smithi089 bash[21353]: audit 2024-04-13T07:23:49.066771+0000 mon.a (mon.0) 334 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:23:49.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:49 smithi089 bash[21353]: audit 2024-04-13T07:23:49.067747+0000 mon.a (mon.0) 335 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:49.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:49 smithi089 bash[31522]: cluster 2024-04-13T07:23:47.847157+0000 mgr.y (mgr.14152) 143 : cluster 0 pgmap v108: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:49.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:49 smithi089 bash[31522]: audit 2024-04-13T07:23:49.063867+0000 mon.a (mon.0) 333 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:23:49.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:49 smithi089 bash[31522]: audit 2024-04-13T07:23:49.066771+0000 mon.a (mon.0) 334 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:23:49.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:49 smithi089 bash[31522]: audit 2024-04-13T07:23:49.067747+0000 mon.a (mon.0) 335 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:23:50.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:50 smithi203 bash[23480]: audit 2024-04-13T07:23:49.060125+0000 mgr.y (mgr.14152) 144 : audit 0 from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:23:50.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:50 smithi089 bash[21353]: audit 2024-04-13T07:23:49.060125+0000 mgr.y (mgr.14152) 144 : audit 0 from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:23:50.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:50 smithi089 bash[31522]: audit 2024-04-13T07:23:49.060125+0000 mgr.y (mgr.14152) 144 : audit 0 from='client.14262 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:23:51.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:51 smithi203 bash[23480]: cluster 2024-04-13T07:23:49.847529+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:51.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:51 smithi089 bash[31522]: cluster 2024-04-13T07:23:49.847529+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:51.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:51 smithi089 bash[21353]: cluster 2024-04-13T07:23:49.847529+0000 mgr.y (mgr.14152) 145 : cluster 0 pgmap v109: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:53.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:53 smithi203 bash[23480]: cluster 2024-04-13T07:23:51.847991+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:53.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:53 smithi089 bash[21353]: cluster 2024-04-13T07:23:51.847991+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:53.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:53 smithi089 bash[31522]: cluster 2024-04-13T07:23:51.847991+0000 mgr.y (mgr.14152) 146 : cluster 0 pgmap v110: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:55.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:55 smithi089 bash[31522]: cluster 2024-04-13T07:23:53.848383+0000 mgr.y (mgr.14152) 147 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:55.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:55 smithi089 bash[21353]: cluster 2024-04-13T07:23:53.848383+0000 mgr.y (mgr.14152) 147 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:55.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:55 smithi203 bash[23480]: cluster 2024-04-13T07:23:53.848383+0000 mgr.y (mgr.14152) 147 : cluster 0 pgmap v111: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:57.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:57 smithi203 bash[23480]: cluster 2024-04-13T07:23:55.848874+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v112: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:57.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:57 smithi203 bash[23480]: audit 2024-04-13T07:23:57.196858+0000 mon.a (mon.0) 336 : audit 1 from='client.? 172.21.15.89:0/1244847875' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f86d6aea-97d3-4d75-99d0-6e5781d7a990"}]: dispatch 2024-04-13T07:23:57.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:57 smithi203 bash[23480]: audit 2024-04-13T07:23:57.202666+0000 mon.a (mon.0) 337 : audit 1 from='client.? 172.21.15.89:0/1244847875' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f86d6aea-97d3-4d75-99d0-6e5781d7a990"}]': finished 2024-04-13T07:23:57.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:57 smithi203 bash[23480]: cluster 2024-04-13T07:23:57.202739+0000 mon.a (mon.0) 338 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-04-13T07:23:57.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:57 smithi203 bash[23480]: audit 2024-04-13T07:23:57.203039+0000 mon.a (mon.0) 339 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:23:57.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[31522]: cluster 2024-04-13T07:23:55.848874+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v112: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[31522]: audit 2024-04-13T07:23:57.196858+0000 mon.a (mon.0) 336 : audit 1 from='client.? 172.21.15.89:0/1244847875' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f86d6aea-97d3-4d75-99d0-6e5781d7a990"}]: dispatch 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[31522]: audit 2024-04-13T07:23:57.202666+0000 mon.a (mon.0) 337 : audit 1 from='client.? 172.21.15.89:0/1244847875' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f86d6aea-97d3-4d75-99d0-6e5781d7a990"}]': finished 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[31522]: cluster 2024-04-13T07:23:57.202739+0000 mon.a (mon.0) 338 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[31522]: audit 2024-04-13T07:23:57.203039+0000 mon.a (mon.0) 339 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[21353]: cluster 2024-04-13T07:23:55.848874+0000 mgr.y (mgr.14152) 148 : cluster 0 pgmap v112: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[21353]: audit 2024-04-13T07:23:57.196858+0000 mon.a (mon.0) 336 : audit 1 from='client.? 172.21.15.89:0/1244847875' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f86d6aea-97d3-4d75-99d0-6e5781d7a990"}]: dispatch 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[21353]: audit 2024-04-13T07:23:57.202666+0000 mon.a (mon.0) 337 : audit 1 from='client.? 172.21.15.89:0/1244847875' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f86d6aea-97d3-4d75-99d0-6e5781d7a990"}]': finished 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[21353]: cluster 2024-04-13T07:23:57.202739+0000 mon.a (mon.0) 338 : cluster 0 osdmap e15: 3 total, 2 up, 3 in 2024-04-13T07:23:57.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:57 smithi089 bash[21353]: audit 2024-04-13T07:23:57.203039+0000 mon.a (mon.0) 339 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:23:58.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:58 smithi203 bash[23480]: audit 2024-04-13T07:23:57.892491+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.89:0/3444441792' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:23:58.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:58 smithi089 bash[21353]: audit 2024-04-13T07:23:57.892491+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.89:0/3444441792' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:23:58.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:58 smithi089 bash[31522]: audit 2024-04-13T07:23:57.892491+0000 mon.c (mon.2) 6 : audit 0 from='client.? 172.21.15.89:0/3444441792' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:23:59.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:23:59 smithi203 bash[23480]: cluster 2024-04-13T07:23:57.849304+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:59.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:23:59 smithi089 bash[21353]: cluster 2024-04-13T07:23:57.849304+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:23:59.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:23:59 smithi089 bash[31522]: cluster 2024-04-13T07:23:57.849304+0000 mgr.y (mgr.14152) 149 : cluster 0 pgmap v114: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:01.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:01 smithi203 bash[23480]: cluster 2024-04-13T07:23:59.849766+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:01.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:01 smithi089 bash[21353]: cluster 2024-04-13T07:23:59.849766+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:01.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:01 smithi089 bash[31522]: cluster 2024-04-13T07:23:59.849766+0000 mgr.y (mgr.14152) 150 : cluster 0 pgmap v115: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:03.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:03 smithi203 bash[23480]: cluster 2024-04-13T07:24:01.850232+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v116: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:03.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:03 smithi089 bash[21353]: cluster 2024-04-13T07:24:01.850232+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v116: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:03.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:03 smithi089 bash[31522]: cluster 2024-04-13T07:24:01.850232+0000 mgr.y (mgr.14152) 151 : cluster 0 pgmap v116: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:05.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:05 smithi203 bash[23480]: cluster 2024-04-13T07:24:03.850566+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:05.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:05 smithi089 bash[21353]: cluster 2024-04-13T07:24:03.850566+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:05.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:05 smithi089 bash[31522]: cluster 2024-04-13T07:24:03.850566+0000 mgr.y (mgr.14152) 152 : cluster 0 pgmap v117: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:07.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:07 smithi203 bash[23480]: cluster 2024-04-13T07:24:05.850885+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:07.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:07 smithi089 bash[21353]: cluster 2024-04-13T07:24:05.850885+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:07.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:07 smithi089 bash[31522]: cluster 2024-04-13T07:24:05.850885+0000 mgr.y (mgr.14152) 153 : cluster 0 pgmap v118: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:09.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:09 smithi203 bash[23480]: cluster 2024-04-13T07:24:07.851315+0000 mgr.y (mgr.14152) 154 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:09.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:09 smithi203 bash[23480]: audit 2024-04-13T07:24:09.104012+0000 mon.a (mon.0) 340 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:24:09.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:09 smithi203 bash[23480]: audit 2024-04-13T07:24:09.104988+0000 mon.a (mon.0) 341 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:09.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:09 smithi089 bash[21353]: cluster 2024-04-13T07:24:07.851315+0000 mgr.y (mgr.14152) 154 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:09.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:09 smithi089 bash[21353]: audit 2024-04-13T07:24:09.104012+0000 mon.a (mon.0) 340 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:24:09.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:09 smithi089 bash[21353]: audit 2024-04-13T07:24:09.104988+0000 mon.a (mon.0) 341 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:09.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:09 smithi089 bash[31522]: cluster 2024-04-13T07:24:07.851315+0000 mgr.y (mgr.14152) 154 : cluster 0 pgmap v119: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:09.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:09 smithi089 bash[31522]: audit 2024-04-13T07:24:09.104012+0000 mon.a (mon.0) 340 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:24:09.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:09 smithi089 bash[31522]: audit 2024-04-13T07:24:09.104988+0000 mon.a (mon.0) 341 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:10.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:10 smithi089 bash[31522]: cephadm 2024-04-13T07:24:09.105770+0000 mgr.y (mgr.14152) 155 : cephadm 1 Deploying daemon osd.2 on smithi089 2024-04-13T07:24:10.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:10 smithi089 bash[21353]: cephadm 2024-04-13T07:24:09.105770+0000 mgr.y (mgr.14152) 155 : cephadm 1 Deploying daemon osd.2 on smithi089 2024-04-13T07:24:10.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:10 smithi203 bash[23480]: cephadm 2024-04-13T07:24:09.105770+0000 mgr.y (mgr.14152) 155 : cephadm 1 Deploying daemon osd.2 on smithi089 2024-04-13T07:24:11.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:11 smithi203 bash[23480]: cluster 2024-04-13T07:24:09.851777+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:11.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:11 smithi089 bash[21353]: cluster 2024-04-13T07:24:09.851777+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:11.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:11 smithi089 bash[31522]: cluster 2024-04-13T07:24:09.851777+0000 mgr.y (mgr.14152) 156 : cluster 0 pgmap v120: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:12.970 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:12 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:12.971 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:24:12 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:12.971 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:24:12 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:12.971 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:12 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:12.972 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:24:12 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:13.236 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:24:13 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:13.236 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:13 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:13.236 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:24:13 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:13.237 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:13 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:13.237 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:24:13 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:24:13.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:13 smithi089 bash[21353]: cluster 2024-04-13T07:24:11.852225+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:13.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:13 smithi089 bash[21353]: audit 2024-04-13T07:24:13.225137+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:13.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:13 smithi089 bash[21353]: audit 2024-04-13T07:24:13.232766+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:13.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:13 smithi089 bash[21353]: audit 2024-04-13T07:24:13.247807+0000 mon.a (mon.0) 344 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:24:13.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:13 smithi089 bash[31522]: cluster 2024-04-13T07:24:11.852225+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:13.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:13 smithi089 bash[31522]: audit 2024-04-13T07:24:13.225137+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:13.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:13 smithi089 bash[31522]: audit 2024-04-13T07:24:13.232766+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:13.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:13 smithi089 bash[31522]: audit 2024-04-13T07:24:13.247807+0000 mon.a (mon.0) 344 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:24:13.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:13 smithi203 bash[23480]: cluster 2024-04-13T07:24:11.852225+0000 mgr.y (mgr.14152) 157 : cluster 0 pgmap v121: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:13.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:13 smithi203 bash[23480]: audit 2024-04-13T07:24:13.225137+0000 mon.a (mon.0) 342 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:13.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:13 smithi203 bash[23480]: audit 2024-04-13T07:24:13.232766+0000 mon.a (mon.0) 343 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:13.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:13 smithi203 bash[23480]: audit 2024-04-13T07:24:13.247807+0000 mon.a (mon.0) 344 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:24:15.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:15 smithi089 bash[21353]: cluster 2024-04-13T07:24:13.852633+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:15.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:15 smithi089 bash[31522]: cluster 2024-04-13T07:24:13.852633+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:15.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:15 smithi203 bash[23480]: cluster 2024-04-13T07:24:13.852633+0000 mgr.y (mgr.14152) 158 : cluster 0 pgmap v122: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:17.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:17 smithi089 bash[21353]: cluster 2024-04-13T07:24:15.853046+0000 mgr.y (mgr.14152) 159 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:17.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:17 smithi089 bash[31522]: cluster 2024-04-13T07:24:15.853046+0000 mgr.y (mgr.14152) 159 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:17.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:17 smithi203 bash[23480]: cluster 2024-04-13T07:24:15.853046+0000 mgr.y (mgr.14152) 159 : cluster 0 pgmap v123: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:19.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[21353]: cluster 2024-04-13T07:24:17.853487+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v124: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:19.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[21353]: audit 2024-04-13T07:24:18.437009+0000 mon.a (mon.0) 345 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:19.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[21353]: audit 2024-04-13T07:24:18.446264+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[21353]: audit 2024-04-13T07:24:18.918434+0000 mon.a (mon.0) 347 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[21353]: audit 2024-04-13T07:24:18.919544+0000 mon.a (mon.0) 348 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[21353]: audit 2024-04-13T07:24:18.926844+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[31522]: cluster 2024-04-13T07:24:17.853487+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v124: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[31522]: audit 2024-04-13T07:24:18.437009+0000 mon.a (mon.0) 345 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[31522]: audit 2024-04-13T07:24:18.446264+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[31522]: audit 2024-04-13T07:24:18.918434+0000 mon.a (mon.0) 347 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[31522]: audit 2024-04-13T07:24:18.919544+0000 mon.a (mon.0) 348 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:24:19.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:19 smithi089 bash[31522]: audit 2024-04-13T07:24:18.926844+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:19.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:19 smithi203 bash[23480]: cluster 2024-04-13T07:24:17.853487+0000 mgr.y (mgr.14152) 160 : cluster 0 pgmap v124: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:19.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:19 smithi203 bash[23480]: audit 2024-04-13T07:24:18.437009+0000 mon.a (mon.0) 345 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:19.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:19 smithi203 bash[23480]: audit 2024-04-13T07:24:18.446264+0000 mon.a (mon.0) 346 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:19.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:19 smithi203 bash[23480]: audit 2024-04-13T07:24:18.918434+0000 mon.a (mon.0) 347 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:19.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:19 smithi203 bash[23480]: audit 2024-04-13T07:24:18.919544+0000 mon.a (mon.0) 348 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:24:19.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:19 smithi203 bash[23480]: audit 2024-04-13T07:24:18.926844+0000 mon.a (mon.0) 349 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:20.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:20 smithi203 bash[23480]: audit 2024-04-13T07:24:19.755384+0000 mon.c (mon.2) 7 : audit 1 from='osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:24:20.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:20 smithi203 bash[23480]: audit 2024-04-13T07:24:19.755778+0000 mon.a (mon.0) 350 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:24:20.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:20 smithi089 bash[21353]: audit 2024-04-13T07:24:19.755384+0000 mon.c (mon.2) 7 : audit 1 from='osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:24:20.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:20 smithi089 bash[21353]: audit 2024-04-13T07:24:19.755778+0000 mon.a (mon.0) 350 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:24:20.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:20 smithi089 bash[31522]: audit 2024-04-13T07:24:19.755384+0000 mon.c (mon.2) 7 : audit 1 from='osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:24:20.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:20 smithi089 bash[31522]: audit 2024-04-13T07:24:19.755778+0000 mon.a (mon.0) 350 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:24:21.665 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 2 on host 'smithi089' 2024-04-13T07:24:21.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[21353]: cluster 2024-04-13T07:24:19.853940+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v125: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:21.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[21353]: audit 2024-04-13T07:24:20.445488+0000 mon.a (mon.0) 351 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-13T07:24:21.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[21353]: cluster 2024-04-13T07:24:20.445589+0000 mon.a (mon.0) 352 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-04-13T07:24:21.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[21353]: audit 2024-04-13T07:24:20.445820+0000 mon.a (mon.0) 353 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:21.714 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[21353]: audit 2024-04-13T07:24:20.449210+0000 mon.c (mon.2) 8 : audit 1 from='osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:24:21.715 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[21353]: audit 2024-04-13T07:24:20.449694+0000 mon.a (mon.0) 354 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:24:21.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[31522]: cluster 2024-04-13T07:24:19.853940+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v125: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:21.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[31522]: audit 2024-04-13T07:24:20.445488+0000 mon.a (mon.0) 351 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-13T07:24:21.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[31522]: cluster 2024-04-13T07:24:20.445589+0000 mon.a (mon.0) 352 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-04-13T07:24:21.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[31522]: audit 2024-04-13T07:24:20.445820+0000 mon.a (mon.0) 353 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:21.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[31522]: audit 2024-04-13T07:24:20.449210+0000 mon.c (mon.2) 8 : audit 1 from='osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:24:21.715 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:21 smithi089 bash[31522]: audit 2024-04-13T07:24:20.449694+0000 mon.a (mon.0) 354 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:24:21.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:21 smithi203 bash[23480]: cluster 2024-04-13T07:24:19.853940+0000 mgr.y (mgr.14152) 161 : cluster 0 pgmap v125: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:21.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:21 smithi203 bash[23480]: audit 2024-04-13T07:24:20.445488+0000 mon.a (mon.0) 351 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-13T07:24:21.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:21 smithi203 bash[23480]: cluster 2024-04-13T07:24:20.445589+0000 mon.a (mon.0) 352 : cluster 0 osdmap e16: 3 total, 2 up, 3 in 2024-04-13T07:24:21.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:21 smithi203 bash[23480]: audit 2024-04-13T07:24:20.445820+0000 mon.a (mon.0) 353 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:21.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:21 smithi203 bash[23480]: audit 2024-04-13T07:24:20.449210+0000 mon.c (mon.2) 8 : audit 1 from='osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:24:21.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:21 smithi203 bash[23480]: audit 2024-04-13T07:24:20.449694+0000 mon.a (mon.0) 354 : audit 1 from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:24:22.341 DEBUG:teuthology.orchestra.run.smithi089:osd.2> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.2.service 2024-04-13T07:24:22.344 INFO:tasks.cephadm:Deploying osd.3 on smithi089 with /dev/vg_nvme/lv_1... 2024-04-13T07:24:22.344 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-13T07:24:22.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:22 smithi203 bash[23480]: audit 2024-04-13T07:24:21.449956+0000 mon.a (mon.0) 355 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:24:22.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:22 smithi203 bash[23480]: cluster 2024-04-13T07:24:21.450104+0000 mon.a (mon.0) 356 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-04-13T07:24:22.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:22 smithi203 bash[23480]: audit 2024-04-13T07:24:21.450624+0000 mon.a (mon.0) 357 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:22.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:22 smithi203 bash[23480]: audit 2024-04-13T07:24:21.462239+0000 mon.a (mon.0) 358 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:22.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:22 smithi203 bash[23480]: audit 2024-04-13T07:24:21.643473+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:22.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:22 smithi203 bash[23480]: audit 2024-04-13T07:24:21.649945+0000 mon.a (mon.0) 360 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:24:22.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:22 smithi203 bash[23480]: audit 2024-04-13T07:24:21.655560+0000 mon.a (mon.0) 361 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:22.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[21353]: audit 2024-04-13T07:24:21.449956+0000 mon.a (mon.0) 355 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:24:22.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[21353]: cluster 2024-04-13T07:24:21.450104+0000 mon.a (mon.0) 356 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[21353]: audit 2024-04-13T07:24:21.450624+0000 mon.a (mon.0) 357 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[21353]: audit 2024-04-13T07:24:21.462239+0000 mon.a (mon.0) 358 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[21353]: audit 2024-04-13T07:24:21.643473+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[21353]: audit 2024-04-13T07:24:21.649945+0000 mon.a (mon.0) 360 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[21353]: audit 2024-04-13T07:24:21.655560+0000 mon.a (mon.0) 361 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[31522]: audit 2024-04-13T07:24:21.449956+0000 mon.a (mon.0) 355 : audit 1 from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[31522]: cluster 2024-04-13T07:24:21.450104+0000 mon.a (mon.0) 356 : cluster 0 osdmap e17: 3 total, 2 up, 3 in 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[31522]: audit 2024-04-13T07:24:21.450624+0000 mon.a (mon.0) 357 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[31522]: audit 2024-04-13T07:24:21.462239+0000 mon.a (mon.0) 358 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[31522]: audit 2024-04-13T07:24:21.643473+0000 mon.a (mon.0) 359 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[31522]: audit 2024-04-13T07:24:21.649945+0000 mon.a (mon.0) 360 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:24:22.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:22 smithi089 bash[31522]: audit 2024-04-13T07:24:21.655560+0000 mon.a (mon.0) 361 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:23.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:23 smithi203 bash[23480]: cluster 2024-04-13T07:24:20.791398+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:24:23.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:23 smithi203 bash[23480]: cluster 2024-04-13T07:24:20.791474+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:24:23.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:23 smithi203 bash[23480]: cluster 2024-04-13T07:24:21.854381+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v128: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:23.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:23 smithi203 bash[23480]: cluster 2024-04-13T07:24:22.464422+0000 mon.a (mon.0) 362 : cluster 1 osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397] boot 2024-04-13T07:24:23.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:23 smithi203 bash[23480]: cluster 2024-04-13T07:24:22.464492+0000 mon.a (mon.0) 363 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-04-13T07:24:23.771 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:23 smithi203 bash[23480]: audit 2024-04-13T07:24:22.465010+0000 mon.a (mon.0) 364 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:23.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[21353]: cluster 2024-04-13T07:24:20.791398+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:24:23.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[21353]: cluster 2024-04-13T07:24:20.791474+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:24:23.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[21353]: cluster 2024-04-13T07:24:21.854381+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v128: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:23.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[21353]: cluster 2024-04-13T07:24:22.464422+0000 mon.a (mon.0) 362 : cluster 1 osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397] boot 2024-04-13T07:24:23.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[21353]: cluster 2024-04-13T07:24:22.464492+0000 mon.a (mon.0) 363 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-04-13T07:24:23.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[21353]: audit 2024-04-13T07:24:22.465010+0000 mon.a (mon.0) 364 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:23.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[31522]: cluster 2024-04-13T07:24:20.791398+0000 osd.2 (osd.2) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:24:23.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[31522]: cluster 2024-04-13T07:24:20.791474+0000 osd.2 (osd.2) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:24:23.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[31522]: cluster 2024-04-13T07:24:21.854381+0000 mgr.y (mgr.14152) 162 : cluster 0 pgmap v128: 0 pgs: ; 0 B data, 53 MiB used, 179 GiB / 179 GiB avail 2024-04-13T07:24:23.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[31522]: cluster 2024-04-13T07:24:22.464422+0000 mon.a (mon.0) 362 : cluster 1 osd.2 [v2:172.21.15.89:6818/46732397,v1:172.21.15.89:6819/46732397] boot 2024-04-13T07:24:23.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[31522]: cluster 2024-04-13T07:24:22.464492+0000 mon.a (mon.0) 363 : cluster 0 osdmap e18: 3 total, 3 up, 3 in 2024-04-13T07:24:23.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:23 smithi089 bash[31522]: audit 2024-04-13T07:24:22.465010+0000 mon.a (mon.0) 364 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:24:25.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:24 smithi203 bash[23480]: cluster 2024-04-13T07:24:23.662033+0000 mon.a (mon.0) 365 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-04-13T07:24:25.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:24 smithi203 bash[23480]: audit 2024-04-13T07:24:23.992047+0000 mon.a (mon.0) 366 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:24:25.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:24 smithi089 bash[31522]: cluster 2024-04-13T07:24:23.662033+0000 mon.a (mon.0) 365 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-04-13T07:24:25.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:24 smithi089 bash[31522]: audit 2024-04-13T07:24:23.992047+0000 mon.a (mon.0) 366 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:24:25.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:24 smithi089 bash[21353]: cluster 2024-04-13T07:24:23.662033+0000 mon.a (mon.0) 365 : cluster 0 osdmap e19: 3 total, 3 up, 3 in 2024-04-13T07:24:25.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:24 smithi089 bash[21353]: audit 2024-04-13T07:24:23.992047+0000 mon.a (mon.0) 366 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:24:26.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:25 smithi203 bash[23480]: cluster 2024-04-13T07:24:23.854944+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v131: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:26.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:25 smithi203 bash[23480]: audit 2024-04-13T07:24:24.672896+0000 mon.a (mon.0) 367 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-13T07:24:26.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:25 smithi203 bash[23480]: cluster 2024-04-13T07:24:24.673050+0000 mon.a (mon.0) 368 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-04-13T07:24:26.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:25 smithi203 bash[23480]: audit 2024-04-13T07:24:24.674877+0000 mon.a (mon.0) 369 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:24:26.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:25 smithi089 bash[21353]: cluster 2024-04-13T07:24:23.854944+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v131: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:26.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:25 smithi089 bash[21353]: audit 2024-04-13T07:24:24.672896+0000 mon.a (mon.0) 367 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-13T07:24:26.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:25 smithi089 bash[21353]: cluster 2024-04-13T07:24:24.673050+0000 mon.a (mon.0) 368 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-04-13T07:24:26.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:25 smithi089 bash[21353]: audit 2024-04-13T07:24:24.674877+0000 mon.a (mon.0) 369 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:24:26.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:25 smithi089 bash[31522]: cluster 2024-04-13T07:24:23.854944+0000 mgr.y (mgr.14152) 163 : cluster 0 pgmap v131: 0 pgs: ; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:26.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:25 smithi089 bash[31522]: audit 2024-04-13T07:24:24.672896+0000 mon.a (mon.0) 367 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32, "yes_i_really_mean_it": true}]': finished 2024-04-13T07:24:26.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:25 smithi089 bash[31522]: cluster 2024-04-13T07:24:24.673050+0000 mon.a (mon.0) 368 : cluster 0 osdmap e20: 3 total, 3 up, 3 in 2024-04-13T07:24:26.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:25 smithi089 bash[31522]: audit 2024-04-13T07:24:24.674877+0000 mon.a (mon.0) 369 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:24:26.495 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:24:26.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:26 smithi089 bash[21353]: audit 2024-04-13T07:24:25.674436+0000 mon.a (mon.0) 370 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-13T07:24:26.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:26 smithi089 bash[21353]: cluster 2024-04-13T07:24:25.674642+0000 mon.a (mon.0) 371 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-04-13T07:24:26.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:26 smithi089 bash[31522]: audit 2024-04-13T07:24:25.674436+0000 mon.a (mon.0) 370 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-13T07:24:26.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:26 smithi089 bash[31522]: cluster 2024-04-13T07:24:25.674642+0000 mon.a (mon.0) 371 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-04-13T07:24:27.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:26 smithi203 bash[23480]: audit 2024-04-13T07:24:25.674436+0000 mon.a (mon.0) 370 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2024-04-13T07:24:27.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:26 smithi203 bash[23480]: cluster 2024-04-13T07:24:25.674642+0000 mon.a (mon.0) 371 : cluster 0 osdmap e21: 3 total, 3 up, 3 in 2024-04-13T07:24:27.990 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:27 smithi089 bash[21353]: cluster 2024-04-13T07:24:25.855519+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v134: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:27.990 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:27 smithi089 bash[21353]: cluster 2024-04-13T07:24:26.691668+0000 mon.a (mon.0) 372 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-04-13T07:24:27.990 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:27 smithi089 bash[21353]: audit 2024-04-13T07:24:26.919949+0000 mon.a (mon.0) 373 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:27.990 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:27 smithi089 bash[31522]: cluster 2024-04-13T07:24:25.855519+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v134: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:27.991 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:27 smithi089 bash[31522]: cluster 2024-04-13T07:24:26.691668+0000 mon.a (mon.0) 372 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-04-13T07:24:27.991 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:27 smithi089 bash[31522]: audit 2024-04-13T07:24:26.919949+0000 mon.a (mon.0) 373 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:28.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:27 smithi203 bash[23480]: cluster 2024-04-13T07:24:25.855519+0000 mgr.y (mgr.14152) 164 : cluster 0 pgmap v134: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:28.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:27 smithi203 bash[23480]: cluster 2024-04-13T07:24:26.691668+0000 mon.a (mon.0) 372 : cluster 0 osdmap e22: 3 total, 3 up, 3 in 2024-04-13T07:24:28.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:27 smithi203 bash[23480]: audit 2024-04-13T07:24:26.919949+0000 mon.a (mon.0) 373 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:30.451 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-13T07:24:30.451 INFO:teuthology.orchestra.run.smithi089.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-13T07:24:30.451 INFO:teuthology.orchestra.run.smithi089.stderr: stderr: 10+0 records in 2024-04-13T07:24:30.451 INFO:teuthology.orchestra.run.smithi089.stderr:10+0 records out 2024-04-13T07:24:30.451 INFO:teuthology.orchestra.run.smithi089.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0128874 s, 814 MB/s 2024-04-13T07:24:30.451 INFO:teuthology.orchestra.run.smithi089.stderr:--> Zapping successful for: 2024-04-13T07:24:30.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: cluster 2024-04-13T07:24:27.855801+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v136: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:30.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: cluster 2024-04-13T07:24:28.708596+0000 mon.a (mon.0) 374 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-04-13T07:24:30.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: audit 2024-04-13T07:24:28.738602+0000 mon.a (mon.0) 375 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:30.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: audit 2024-04-13T07:24:29.642257+0000 mon.a (mon.0) 376 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:30.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: audit 2024-04-13T07:24:29.642834+0000 mon.a (mon.0) 377 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: audit 2024-04-13T07:24:29.643264+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: audit 2024-04-13T07:24:29.643423+0000 mon.a (mon.0) 379 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: audit 2024-04-13T07:24:29.646306+0000 mon.a (mon.0) 380 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: audit 2024-04-13T07:24:29.646537+0000 mon.a (mon.0) 381 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[21353]: audit 2024-04-13T07:24:29.646694+0000 mon.a (mon.0) 382 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: cluster 2024-04-13T07:24:27.855801+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v136: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: cluster 2024-04-13T07:24:28.708596+0000 mon.a (mon.0) 374 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: audit 2024-04-13T07:24:28.738602+0000 mon.a (mon.0) 375 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: audit 2024-04-13T07:24:29.642257+0000 mon.a (mon.0) 376 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: audit 2024-04-13T07:24:29.642834+0000 mon.a (mon.0) 377 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:30.804 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: audit 2024-04-13T07:24:29.643264+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:30.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: audit 2024-04-13T07:24:29.643423+0000 mon.a (mon.0) 379 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:30.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: audit 2024-04-13T07:24:29.646306+0000 mon.a (mon.0) 380 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:30.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: audit 2024-04-13T07:24:29.646537+0000 mon.a (mon.0) 381 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:30.805 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:30 smithi089 bash[31522]: audit 2024-04-13T07:24:29.646694+0000 mon.a (mon.0) 382 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:31.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: cluster 2024-04-13T07:24:27.855801+0000 mgr.y (mgr.14152) 165 : cluster 0 pgmap v136: 1 pgs: 1 unknown; 0 B data, 80 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:31.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: cluster 2024-04-13T07:24:28.708596+0000 mon.a (mon.0) 374 : cluster 0 mgrmap e15: y(active, since 4m), standbys: x 2024-04-13T07:24:31.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: audit 2024-04-13T07:24:28.738602+0000 mon.a (mon.0) 375 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:31.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: audit 2024-04-13T07:24:29.642257+0000 mon.a (mon.0) 376 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:31.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: audit 2024-04-13T07:24:29.642834+0000 mon.a (mon.0) 377 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:31.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: audit 2024-04-13T07:24:29.643264+0000 mon.a (mon.0) 378 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:31.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: audit 2024-04-13T07:24:29.643423+0000 mon.a (mon.0) 379 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:31.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: audit 2024-04-13T07:24:29.646306+0000 mon.a (mon.0) 380 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:31.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: audit 2024-04-13T07:24:29.646537+0000 mon.a (mon.0) 381 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:31.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:30 smithi203 bash[23480]: audit 2024-04-13T07:24:29.646694+0000 mon.a (mon.0) 382 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:31.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi089:vg_nvme/lv_1 2024-04-13T07:24:32.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: audit 2024-04-13T07:24:29.645532+0000 mon.b (mon.1) 7 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:32.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: cluster 2024-04-13T07:24:29.856098+0000 mgr.y (mgr.14152) 166 : cluster 0 pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:32.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: audit 2024-04-13T07:24:30.553417+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:32.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: audit 2024-04-13T07:24:30.578538+0000 mon.a (mon.0) 383 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:32.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: audit 2024-04-13T07:24:30.580931+0000 mon.c (mon.2) 9 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:32.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: audit 2024-04-13T07:24:30.581102+0000 mon.a (mon.0) 384 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:32.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: audit 2024-04-13T07:24:30.581246+0000 mon.a (mon.0) 385 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:32.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: audit 2024-04-13T07:24:30.581514+0000 mon.a (mon.0) 386 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:32.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:31 smithi203 bash[23480]: audit 2024-04-13T07:24:31.465683+0000 mon.c (mon.2) 10 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:32.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: audit 2024-04-13T07:24:29.645532+0000 mon.b (mon.1) 7 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: cluster 2024-04-13T07:24:29.856098+0000 mgr.y (mgr.14152) 166 : cluster 0 pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: audit 2024-04-13T07:24:30.553417+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: audit 2024-04-13T07:24:30.578538+0000 mon.a (mon.0) 383 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: audit 2024-04-13T07:24:30.580931+0000 mon.c (mon.2) 9 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: audit 2024-04-13T07:24:30.581102+0000 mon.a (mon.0) 384 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: audit 2024-04-13T07:24:30.581246+0000 mon.a (mon.0) 385 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: audit 2024-04-13T07:24:30.581514+0000 mon.a (mon.0) 386 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[21353]: audit 2024-04-13T07:24:31.465683+0000 mon.c (mon.2) 10 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:32.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: audit 2024-04-13T07:24:29.645532+0000 mon.b (mon.1) 7 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: cluster 2024-04-13T07:24:29.856098+0000 mgr.y (mgr.14152) 166 : cluster 0 pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: audit 2024-04-13T07:24:30.553417+0000 mon.b (mon.1) 8 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: audit 2024-04-13T07:24:30.578538+0000 mon.a (mon.0) 383 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: audit 2024-04-13T07:24:30.580931+0000 mon.c (mon.2) 9 : audit 1 from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2024-04-13T07:24:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: audit 2024-04-13T07:24:30.581102+0000 mon.a (mon.0) 384 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:24:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: audit 2024-04-13T07:24:30.581246+0000 mon.a (mon.0) 385 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:24:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: audit 2024-04-13T07:24:30.581514+0000 mon.a (mon.0) 386 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:24:32.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:31 smithi089 bash[31522]: audit 2024-04-13T07:24:31.465683+0000 mon.c (mon.2) 10 : audit 1 from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2024-04-13T07:24:33.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: cephadm 2024-04-13T07:24:31.595399+0000 mgr.y (mgr.14152) 167 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:24:33.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.607416+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.621332+0000 mon.a (mon.0) 388 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.623808+0000 mon.a (mon.0) 389 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.625522+0000 mon.a (mon.0) 390 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.627095+0000 mon.a (mon.0) 391 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: cephadm 2024-04-13T07:24:31.628363+0000 mgr.y (mgr.14152) 168 : cephadm 1 Adjusting osd_memory_target on smithi089 to 5413M 2024-04-13T07:24:33.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.639517+0000 mon.a (mon.0) 392 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.641597+0000 mon.a (mon.0) 393 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:33.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.643430+0000 mon.a (mon.0) 394 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:24:33.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:32 smithi203 bash[23480]: audit 2024-04-13T07:24:31.653394+0000 mon.a (mon.0) 395 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: cephadm 2024-04-13T07:24:31.595399+0000 mgr.y (mgr.14152) 167 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.607416+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.621332+0000 mon.a (mon.0) 388 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.623808+0000 mon.a (mon.0) 389 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.625522+0000 mon.a (mon.0) 390 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.627095+0000 mon.a (mon.0) 391 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: cephadm 2024-04-13T07:24:31.628363+0000 mgr.y (mgr.14152) 168 : cephadm 1 Adjusting osd_memory_target on smithi089 to 5413M 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.639517+0000 mon.a (mon.0) 392 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.641597+0000 mon.a (mon.0) 393 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.643430+0000 mon.a (mon.0) 394 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[21353]: audit 2024-04-13T07:24:31.653394+0000 mon.a (mon.0) 395 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: cephadm 2024-04-13T07:24:31.595399+0000 mgr.y (mgr.14152) 167 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.607416+0000 mon.a (mon.0) 387 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.621332+0000 mon.a (mon.0) 388 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.623808+0000 mon.a (mon.0) 389 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.625522+0000 mon.a (mon.0) 390 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.627095+0000 mon.a (mon.0) 391 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: cephadm 2024-04-13T07:24:31.628363+0000 mgr.y (mgr.14152) 168 : cephadm 1 Adjusting osd_memory_target on smithi089 to 5413M 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.639517+0000 mon.a (mon.0) 392 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.641597+0000 mon.a (mon.0) 393 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.643430+0000 mon.a (mon.0) 394 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:24:33.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:32 smithi089 bash[31522]: audit 2024-04-13T07:24:31.653394+0000 mon.a (mon.0) 395 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:24:34.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:33 smithi203 bash[23480]: cluster 2024-04-13T07:24:31.856565+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:34.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:33 smithi089 bash[21353]: cluster 2024-04-13T07:24:31.856565+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:34.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:33 smithi089 bash[31522]: cluster 2024-04-13T07:24:31.856565+0000 mgr.y (mgr.14152) 169 : cluster 0 pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:35.858 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:24:36.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:35 smithi203 bash[23480]: cluster 2024-04-13T07:24:33.856931+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:36.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:35 smithi089 bash[21353]: cluster 2024-04-13T07:24:33.856931+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:36.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:35 smithi089 bash[31522]: cluster 2024-04-13T07:24:33.856931+0000 mgr.y (mgr.14152) 170 : cluster 0 pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:37.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:37 smithi089 bash[21353]: cluster 2024-04-13T07:24:35.857514+0000 mgr.y (mgr.14152) 171 : cluster 0 pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-13T07:24:37.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:37 smithi089 bash[31522]: cluster 2024-04-13T07:24:35.857514+0000 mgr.y (mgr.14152) 171 : cluster 0 pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-13T07:24:38.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:37 smithi203 bash[23480]: cluster 2024-04-13T07:24:35.857514+0000 mgr.y (mgr.14152) 171 : cluster 0 pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2024-04-13T07:24:39.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:38 smithi203 bash[23480]: audit 2024-04-13T07:24:38.444081+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:24:39.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:38 smithi203 bash[23480]: audit 2024-04-13T07:24:38.448076+0000 mon.a (mon.0) 397 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:24:39.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:38 smithi203 bash[23480]: audit 2024-04-13T07:24:38.448977+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:39.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:38 smithi089 bash[21353]: audit 2024-04-13T07:24:38.444081+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:24:39.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:38 smithi089 bash[21353]: audit 2024-04-13T07:24:38.448076+0000 mon.a (mon.0) 397 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:24:39.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:38 smithi089 bash[21353]: audit 2024-04-13T07:24:38.448977+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:39.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:38 smithi089 bash[31522]: audit 2024-04-13T07:24:38.444081+0000 mon.a (mon.0) 396 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:24:39.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:38 smithi089 bash[31522]: audit 2024-04-13T07:24:38.448076+0000 mon.a (mon.0) 397 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:24:39.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:38 smithi089 bash[31522]: audit 2024-04-13T07:24:38.448977+0000 mon.a (mon.0) 398 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:40.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:39 smithi203 bash[23480]: cluster 2024-04-13T07:24:37.858039+0000 mgr.y (mgr.14152) 172 : cluster 0 pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-13T07:24:40.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:39 smithi203 bash[23480]: audit 2024-04-13T07:24:38.437502+0000 mgr.y (mgr.14152) 173 : audit 0 from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:24:40.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:39 smithi089 bash[21353]: cluster 2024-04-13T07:24:37.858039+0000 mgr.y (mgr.14152) 172 : cluster 0 pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-13T07:24:40.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:39 smithi089 bash[21353]: audit 2024-04-13T07:24:38.437502+0000 mgr.y (mgr.14152) 173 : audit 0 from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:24:40.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:39 smithi089 bash[31522]: cluster 2024-04-13T07:24:37.858039+0000 mgr.y (mgr.14152) 172 : cluster 0 pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2024-04-13T07:24:40.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:39 smithi089 bash[31522]: audit 2024-04-13T07:24:38.437502+0000 mgr.y (mgr.14152) 173 : audit 0 from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi089:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:24:42.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:41 smithi203 bash[23480]: cluster 2024-04-13T07:24:39.858592+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-13T07:24:42.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:41 smithi089 bash[21353]: cluster 2024-04-13T07:24:39.858592+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-13T07:24:42.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:41 smithi089 bash[31522]: cluster 2024-04-13T07:24:39.858592+0000 mgr.y (mgr.14152) 174 : cluster 0 pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2024-04-13T07:24:44.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:43 smithi203 bash[23480]: cluster 2024-04-13T07:24:41.859157+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:44.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:43 smithi089 bash[21353]: cluster 2024-04-13T07:24:41.859157+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:44.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:43 smithi089 bash[31522]: cluster 2024-04-13T07:24:41.859157+0000 mgr.y (mgr.14152) 175 : cluster 0 pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:45 smithi089 bash[21353]: cluster 2024-04-13T07:24:43.859730+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:45.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:45 smithi089 bash[31522]: cluster 2024-04-13T07:24:43.859730+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:46.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:45 smithi203 bash[23480]: cluster 2024-04-13T07:24:43.859730+0000 mgr.y (mgr.14152) 176 : cluster 0 pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:47.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:46 smithi203 bash[23480]: audit 2024-04-13T07:24:46.451757+0000 mon.a (mon.0) 399 : audit 1 from='client.? 172.21.15.89:0/3335204052' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "37c263d3-d642-4cfa-99da-5a33e1bef16e"}]: dispatch 2024-04-13T07:24:47.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:46 smithi203 bash[23480]: audit 2024-04-13T07:24:46.462479+0000 mon.a (mon.0) 400 : audit 1 from='client.? 172.21.15.89:0/3335204052' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "37c263d3-d642-4cfa-99da-5a33e1bef16e"}]': finished 2024-04-13T07:24:47.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:46 smithi203 bash[23480]: cluster 2024-04-13T07:24:46.462665+0000 mon.a (mon.0) 401 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-04-13T07:24:47.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:46 smithi203 bash[23480]: audit 2024-04-13T07:24:46.462971+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:24:47.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:46 smithi089 bash[31522]: audit 2024-04-13T07:24:46.451757+0000 mon.a (mon.0) 399 : audit 1 from='client.? 172.21.15.89:0/3335204052' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "37c263d3-d642-4cfa-99da-5a33e1bef16e"}]: dispatch 2024-04-13T07:24:47.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:46 smithi089 bash[31522]: audit 2024-04-13T07:24:46.462479+0000 mon.a (mon.0) 400 : audit 1 from='client.? 172.21.15.89:0/3335204052' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "37c263d3-d642-4cfa-99da-5a33e1bef16e"}]': finished 2024-04-13T07:24:47.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:46 smithi089 bash[31522]: cluster 2024-04-13T07:24:46.462665+0000 mon.a (mon.0) 401 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-04-13T07:24:47.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:46 smithi089 bash[31522]: audit 2024-04-13T07:24:46.462971+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:24:47.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:46 smithi089 bash[21353]: audit 2024-04-13T07:24:46.451757+0000 mon.a (mon.0) 399 : audit 1 from='client.? 172.21.15.89:0/3335204052' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "37c263d3-d642-4cfa-99da-5a33e1bef16e"}]: dispatch 2024-04-13T07:24:47.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:46 smithi089 bash[21353]: audit 2024-04-13T07:24:46.462479+0000 mon.a (mon.0) 400 : audit 1 from='client.? 172.21.15.89:0/3335204052' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "37c263d3-d642-4cfa-99da-5a33e1bef16e"}]': finished 2024-04-13T07:24:47.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:46 smithi089 bash[21353]: cluster 2024-04-13T07:24:46.462665+0000 mon.a (mon.0) 401 : cluster 0 osdmap e23: 4 total, 3 up, 4 in 2024-04-13T07:24:47.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:46 smithi089 bash[21353]: audit 2024-04-13T07:24:46.462971+0000 mon.a (mon.0) 402 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:24:48.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:47 smithi203 bash[23480]: cluster 2024-04-13T07:24:45.860108+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:48.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:47 smithi203 bash[23480]: audit 2024-04-13T07:24:47.155327+0000 mon.a (mon.0) 403 : audit 0 from='client.? 172.21.15.89:0/4009785941' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:24:48.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:47 smithi089 bash[31522]: cluster 2024-04-13T07:24:45.860108+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:48.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:47 smithi089 bash[31522]: audit 2024-04-13T07:24:47.155327+0000 mon.a (mon.0) 403 : audit 0 from='client.? 172.21.15.89:0/4009785941' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:24:48.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:47 smithi089 bash[21353]: cluster 2024-04-13T07:24:45.860108+0000 mgr.y (mgr.14152) 177 : cluster 0 pgmap v145: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail; 1.5 KiB/s rd, 65 KiB/s wr, 4 op/s 2024-04-13T07:24:48.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:47 smithi089 bash[21353]: audit 2024-04-13T07:24:47.155327+0000 mon.a (mon.0) 403 : audit 0 from='client.? 172.21.15.89:0/4009785941' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:24:50.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:49 smithi203 bash[23480]: cluster 2024-04-13T07:24:47.860590+0000 mgr.y (mgr.14152) 178 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:50.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:49 smithi089 bash[31522]: cluster 2024-04-13T07:24:47.860590+0000 mgr.y (mgr.14152) 178 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:50.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:49 smithi089 bash[21353]: cluster 2024-04-13T07:24:47.860590+0000 mgr.y (mgr.14152) 178 : cluster 0 pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 82 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:52.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:51 smithi203 bash[23480]: cluster 2024-04-13T07:24:49.861128+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:52.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:51 smithi089 bash[21353]: cluster 2024-04-13T07:24:49.861128+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:52.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:51 smithi089 bash[31522]: cluster 2024-04-13T07:24:49.861128+0000 mgr.y (mgr.14152) 179 : cluster 0 pgmap v148: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:54.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:53 smithi203 bash[23480]: cluster 2024-04-13T07:24:51.861724+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:53 smithi089 bash[21353]: cluster 2024-04-13T07:24:51.861724+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:54.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:53 smithi089 bash[31522]: cluster 2024-04-13T07:24:51.861724+0000 mgr.y (mgr.14152) 180 : cluster 0 pgmap v149: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:55.985 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:55 smithi089 bash[21353]: cluster 2024-04-13T07:24:53.862326+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:55.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:55 smithi089 bash[31522]: cluster 2024-04-13T07:24:53.862326+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:56.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:55 smithi203 bash[23480]: cluster 2024-04-13T07:24:53.862326+0000 mgr.y (mgr.14152) 181 : cluster 0 pgmap v150: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:58.015 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:57 smithi089 bash[31522]: cluster 2024-04-13T07:24:55.862912+0000 mgr.y (mgr.14152) 182 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:58.015 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:57 smithi089 bash[21353]: cluster 2024-04-13T07:24:55.862912+0000 mgr.y (mgr.14152) 182 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:58.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:57 smithi203 bash[23480]: cluster 2024-04-13T07:24:55.862912+0000 mgr.y (mgr.14152) 182 : cluster 0 pgmap v151: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:59.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:58 smithi203 bash[23480]: audit 2024-04-13T07:24:58.460646+0000 mon.a (mon.0) 404 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:24:59.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:58 smithi203 bash[23480]: audit 2024-04-13T07:24:58.462155+0000 mon.a (mon.0) 405 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:59.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:58 smithi089 bash[21353]: audit 2024-04-13T07:24:58.460646+0000 mon.a (mon.0) 404 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:24:59.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:58 smithi089 bash[21353]: audit 2024-04-13T07:24:58.462155+0000 mon.a (mon.0) 405 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:59.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:58 smithi089 bash[31522]: audit 2024-04-13T07:24:58.460646+0000 mon.a (mon.0) 404 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:24:59.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:58 smithi089 bash[31522]: audit 2024-04-13T07:24:58.462155+0000 mon.a (mon.0) 405 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:24:59.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:59 smithi089 bash[21353]: cluster 2024-04-13T07:24:57.863415+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:59.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:24:59 smithi089 bash[21353]: cephadm 2024-04-13T07:24:58.463741+0000 mgr.y (mgr.14152) 184 : cephadm 1 Deploying daemon osd.3 on smithi089 2024-04-13T07:24:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:59 smithi089 bash[31522]: cluster 2024-04-13T07:24:57.863415+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:24:59.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:24:59 smithi089 bash[31522]: cephadm 2024-04-13T07:24:58.463741+0000 mgr.y (mgr.14152) 184 : cephadm 1 Deploying daemon osd.3 on smithi089 2024-04-13T07:25:00.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:59 smithi203 bash[23480]: cluster 2024-04-13T07:24:57.863415+0000 mgr.y (mgr.14152) 183 : cluster 0 pgmap v152: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:00.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:24:59 smithi203 bash[23480]: cephadm 2024-04-13T07:24:58.463741+0000 mgr.y (mgr.14152) 184 : cephadm 1 Deploying daemon osd.3 on smithi089 2024-04-13T07:25:02.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:01 smithi203 bash[23480]: cluster 2024-04-13T07:24:59.863921+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:01 smithi089 bash[21353]: cluster 2024-04-13T07:24:59.863921+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:02.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:01 smithi089 bash[31522]: cluster 2024-04-13T07:24:59.863921+0000 mgr.y (mgr.14152) 185 : cluster 0 pgmap v153: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:02.458 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.458 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.459 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.459 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.459 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.460 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.803 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.803 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.804 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:02.804 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:25:02 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:04.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:03 smithi203 bash[23480]: cluster 2024-04-13T07:25:01.864442+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:04.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:03 smithi203 bash[23480]: audit 2024-04-13T07:25:02.727431+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:04.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:03 smithi203 bash[23480]: audit 2024-04-13T07:25:02.734175+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:04.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:03 smithi203 bash[23480]: audit 2024-04-13T07:25:02.764344+0000 mon.a (mon.0) 408 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:04.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:03 smithi089 bash[31522]: cluster 2024-04-13T07:25:01.864442+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:04.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:03 smithi089 bash[31522]: audit 2024-04-13T07:25:02.727431+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:04.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:03 smithi089 bash[31522]: audit 2024-04-13T07:25:02.734175+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:04.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:03 smithi089 bash[31522]: audit 2024-04-13T07:25:02.764344+0000 mon.a (mon.0) 408 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:04.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:03 smithi089 bash[21353]: cluster 2024-04-13T07:25:01.864442+0000 mgr.y (mgr.14152) 186 : cluster 0 pgmap v154: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:04.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:03 smithi089 bash[21353]: audit 2024-04-13T07:25:02.727431+0000 mon.a (mon.0) 406 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:04.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:03 smithi089 bash[21353]: audit 2024-04-13T07:25:02.734175+0000 mon.a (mon.0) 407 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:04.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:03 smithi089 bash[21353]: audit 2024-04-13T07:25:02.764344+0000 mon.a (mon.0) 408 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:06.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:05 smithi203 bash[23480]: cluster 2024-04-13T07:25:03.864959+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:06.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:05 smithi089 bash[31522]: cluster 2024-04-13T07:25:03.864959+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:06.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:05 smithi089 bash[21353]: cluster 2024-04-13T07:25:03.864959+0000 mgr.y (mgr.14152) 187 : cluster 0 pgmap v155: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:08.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:07 smithi203 bash[23480]: cluster 2024-04-13T07:25:05.865508+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:08.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:07 smithi089 bash[21353]: cluster 2024-04-13T07:25:05.865508+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:08.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:07 smithi089 bash[31522]: cluster 2024-04-13T07:25:05.865508+0000 mgr.y (mgr.14152) 188 : cluster 0 pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:09.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[31522]: cluster 2024-04-13T07:25:07.865833+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:09.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[31522]: audit 2024-04-13T07:25:07.908439+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:09.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[31522]: audit 2024-04-13T07:25:07.914979+0000 mon.a (mon.0) 410 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:09.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[31522]: audit 2024-04-13T07:25:07.915885+0000 mon.a (mon.0) 411 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:09.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[31522]: audit 2024-04-13T07:25:07.916614+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:09.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[31522]: audit 2024-04-13T07:25:07.923379+0000 mon.a (mon.0) 413 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:09.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[21353]: cluster 2024-04-13T07:25:07.865833+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:09.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[21353]: audit 2024-04-13T07:25:07.908439+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:09.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[21353]: audit 2024-04-13T07:25:07.914979+0000 mon.a (mon.0) 410 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:09.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[21353]: audit 2024-04-13T07:25:07.915885+0000 mon.a (mon.0) 411 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:09.072 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[21353]: audit 2024-04-13T07:25:07.916614+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:09.073 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:08 smithi089 bash[21353]: audit 2024-04-13T07:25:07.923379+0000 mon.a (mon.0) 413 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:09.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:08 smithi203 bash[23480]: cluster 2024-04-13T07:25:07.865833+0000 mgr.y (mgr.14152) 189 : cluster 0 pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:09.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:08 smithi203 bash[23480]: audit 2024-04-13T07:25:07.908439+0000 mon.a (mon.0) 409 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:09.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:08 smithi203 bash[23480]: audit 2024-04-13T07:25:07.914979+0000 mon.a (mon.0) 410 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:09.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:08 smithi203 bash[23480]: audit 2024-04-13T07:25:07.915885+0000 mon.a (mon.0) 411 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:09.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:08 smithi203 bash[23480]: audit 2024-04-13T07:25:07.916614+0000 mon.a (mon.0) 412 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:09.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:08 smithi203 bash[23480]: audit 2024-04-13T07:25:07.923379+0000 mon.a (mon.0) 413 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:10.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:09 smithi203 bash[23480]: audit 2024-04-13T07:25:09.076285+0000 mon.c (mon.2) 11 : audit 1 from='osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:25:10.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:09 smithi203 bash[23480]: audit 2024-04-13T07:25:09.076661+0000 mon.a (mon.0) 414 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:25:10.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:09 smithi089 bash[21353]: audit 2024-04-13T07:25:09.076285+0000 mon.c (mon.2) 11 : audit 1 from='osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:25:10.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:09 smithi089 bash[21353]: audit 2024-04-13T07:25:09.076661+0000 mon.a (mon.0) 414 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:25:10.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:09 smithi089 bash[31522]: audit 2024-04-13T07:25:09.076285+0000 mon.c (mon.2) 11 : audit 1 from='osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:25:10.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:09 smithi089 bash[31522]: audit 2024-04-13T07:25:09.076661+0000 mon.a (mon.0) 414 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:25:11.144 INFO:teuthology.orchestra.run.smithi089.stdout:Created osd(s) 3 on host 'smithi089' 2024-04-13T07:25:11.201 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[21353]: cluster 2024-04-13T07:25:09.866384+0000 mgr.y (mgr.14152) 190 : cluster 0 pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:11.201 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[21353]: audit 2024-04-13T07:25:09.926462+0000 mon.a (mon.0) 415 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-13T07:25:11.201 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[21353]: cluster 2024-04-13T07:25:09.926584+0000 mon.a (mon.0) 416 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-04-13T07:25:11.201 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[21353]: audit 2024-04-13T07:25:09.926774+0000 mon.a (mon.0) 417 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:11.201 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[21353]: audit 2024-04-13T07:25:09.927906+0000 mon.c (mon.2) 12 : audit 1 from='osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:25:11.202 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[21353]: audit 2024-04-13T07:25:09.928366+0000 mon.a (mon.0) 418 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:25:11.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[31522]: cluster 2024-04-13T07:25:09.866384+0000 mgr.y (mgr.14152) 190 : cluster 0 pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:11.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[31522]: audit 2024-04-13T07:25:09.926462+0000 mon.a (mon.0) 415 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-13T07:25:11.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[31522]: cluster 2024-04-13T07:25:09.926584+0000 mon.a (mon.0) 416 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-04-13T07:25:11.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[31522]: audit 2024-04-13T07:25:09.926774+0000 mon.a (mon.0) 417 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:11.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[31522]: audit 2024-04-13T07:25:09.927906+0000 mon.c (mon.2) 12 : audit 1 from='osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:25:11.202 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:10 smithi089 bash[31522]: audit 2024-04-13T07:25:09.928366+0000 mon.a (mon.0) 418 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:25:11.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:10 smithi203 bash[23480]: cluster 2024-04-13T07:25:09.866384+0000 mgr.y (mgr.14152) 190 : cluster 0 pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:11.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:10 smithi203 bash[23480]: audit 2024-04-13T07:25:09.926462+0000 mon.a (mon.0) 415 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-13T07:25:11.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:10 smithi203 bash[23480]: cluster 2024-04-13T07:25:09.926584+0000 mon.a (mon.0) 416 : cluster 0 osdmap e24: 4 total, 3 up, 4 in 2024-04-13T07:25:11.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:10 smithi203 bash[23480]: audit 2024-04-13T07:25:09.926774+0000 mon.a (mon.0) 417 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:11.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:10 smithi203 bash[23480]: audit 2024-04-13T07:25:09.927906+0000 mon.c (mon.2) 12 : audit 1 from='osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:25:11.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:10 smithi203 bash[23480]: audit 2024-04-13T07:25:09.928366+0000 mon.a (mon.0) 418 : audit 1 from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:25:11.778 DEBUG:teuthology.orchestra.run.smithi089:osd.3> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.3.service 2024-04-13T07:25:11.780 INFO:tasks.cephadm:Deploying osd.4 on smithi203 with /dev/vg_nvme/lv_4... 2024-04-13T07:25:11.781 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[21353]: audit 2024-04-13T07:25:10.929282+0000 mon.a (mon.0) 419 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[21353]: cluster 2024-04-13T07:25:10.929414+0000 mon.a (mon.0) 420 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[21353]: audit 2024-04-13T07:25:10.931120+0000 mon.a (mon.0) 421 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[21353]: audit 2024-04-13T07:25:10.945691+0000 mon.a (mon.0) 422 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[21353]: audit 2024-04-13T07:25:11.120964+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[21353]: audit 2024-04-13T07:25:11.129321+0000 mon.a (mon.0) 424 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[21353]: audit 2024-04-13T07:25:11.145342+0000 mon.a (mon.0) 425 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[31522]: audit 2024-04-13T07:25:10.929282+0000 mon.a (mon.0) 419 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:25:12.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[31522]: cluster 2024-04-13T07:25:10.929414+0000 mon.a (mon.0) 420 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-04-13T07:25:12.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[31522]: audit 2024-04-13T07:25:10.931120+0000 mon.a (mon.0) 421 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:12.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[31522]: audit 2024-04-13T07:25:10.945691+0000 mon.a (mon.0) 422 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:12.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[31522]: audit 2024-04-13T07:25:11.120964+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:12.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[31522]: audit 2024-04-13T07:25:11.129321+0000 mon.a (mon.0) 424 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:12.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:11 smithi089 bash[31522]: audit 2024-04-13T07:25:11.145342+0000 mon.a (mon.0) 425 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:12.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:11 smithi203 bash[23480]: audit 2024-04-13T07:25:10.929282+0000 mon.a (mon.0) 419 : audit 1 from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]': finished 2024-04-13T07:25:12.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:11 smithi203 bash[23480]: cluster 2024-04-13T07:25:10.929414+0000 mon.a (mon.0) 420 : cluster 0 osdmap e25: 4 total, 3 up, 4 in 2024-04-13T07:25:12.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:11 smithi203 bash[23480]: audit 2024-04-13T07:25:10.931120+0000 mon.a (mon.0) 421 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:12.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:11 smithi203 bash[23480]: audit 2024-04-13T07:25:10.945691+0000 mon.a (mon.0) 422 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:12.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:11 smithi203 bash[23480]: audit 2024-04-13T07:25:11.120964+0000 mon.a (mon.0) 423 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:12.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:11 smithi203 bash[23480]: audit 2024-04-13T07:25:11.129321+0000 mon.a (mon.0) 424 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:12.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:11 smithi203 bash[23480]: audit 2024-04-13T07:25:11.145342+0000 mon.a (mon.0) 425 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:13.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:12 smithi203 bash[23480]: cluster 2024-04-13T07:25:10.117545+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:25:13.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:12 smithi203 bash[23480]: cluster 2024-04-13T07:25:10.117628+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:25:13.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:12 smithi203 bash[23480]: cluster 2024-04-13T07:25:11.866922+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:13.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:12 smithi203 bash[23480]: cluster 2024-04-13T07:25:11.946424+0000 mon.a (mon.0) 426 : cluster 1 osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060] boot 2024-04-13T07:25:13.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:12 smithi203 bash[23480]: cluster 2024-04-13T07:25:11.946527+0000 mon.a (mon.0) 427 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-04-13T07:25:13.271 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:12 smithi203 bash[23480]: audit 2024-04-13T07:25:11.947309+0000 mon.a (mon.0) 428 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:13.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[21353]: cluster 2024-04-13T07:25:10.117545+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:25:13.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[21353]: cluster 2024-04-13T07:25:10.117628+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:25:13.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[21353]: cluster 2024-04-13T07:25:11.866922+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:13.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[21353]: cluster 2024-04-13T07:25:11.946424+0000 mon.a (mon.0) 426 : cluster 1 osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060] boot 2024-04-13T07:25:13.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[21353]: cluster 2024-04-13T07:25:11.946527+0000 mon.a (mon.0) 427 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-04-13T07:25:13.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[21353]: audit 2024-04-13T07:25:11.947309+0000 mon.a (mon.0) 428 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:13.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[31522]: cluster 2024-04-13T07:25:10.117545+0000 osd.3 (osd.3) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:25:13.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[31522]: cluster 2024-04-13T07:25:10.117628+0000 osd.3 (osd.3) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:25:13.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[31522]: cluster 2024-04-13T07:25:11.866922+0000 mgr.y (mgr.14152) 191 : cluster 0 pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 81 MiB used, 268 GiB / 268 GiB avail 2024-04-13T07:25:13.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[31522]: cluster 2024-04-13T07:25:11.946424+0000 mon.a (mon.0) 426 : cluster 1 osd.3 [v2:172.21.15.89:6826/3306593060,v1:172.21.15.89:6827/3306593060] boot 2024-04-13T07:25:13.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[31522]: cluster 2024-04-13T07:25:11.946527+0000 mon.a (mon.0) 427 : cluster 0 osdmap e26: 4 total, 4 up, 4 in 2024-04-13T07:25:13.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:12 smithi089 bash[31522]: audit 2024-04-13T07:25:11.947309+0000 mon.a (mon.0) 428 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:25:14.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:13 smithi203 bash[23480]: cluster 2024-04-13T07:25:12.963252+0000 mon.a (mon.0) 429 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-04-13T07:25:14.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:13 smithi089 bash[21353]: cluster 2024-04-13T07:25:12.963252+0000 mon.a (mon.0) 429 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-04-13T07:25:14.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:13 smithi089 bash[31522]: cluster 2024-04-13T07:25:12.963252+0000 mon.a (mon.0) 429 : cluster 0 osdmap e27: 4 total, 4 up, 4 in 2024-04-13T07:25:15.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:14 smithi203 bash[23480]: cluster 2024-04-13T07:25:13.867429+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:15.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:14 smithi089 bash[21353]: cluster 2024-04-13T07:25:13.867429+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:15.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:14 smithi089 bash[31522]: cluster 2024-04-13T07:25:13.867429+0000 mgr.y (mgr.14152) 192 : cluster 0 pgmap v164: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:15.443 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:25:17.162 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:16 smithi089 bash[31522]: cluster 2024-04-13T07:25:15.867990+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:17.163 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:16 smithi089 bash[21353]: cluster 2024-04-13T07:25:15.867990+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:17.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:16 smithi203 bash[23480]: cluster 2024-04-13T07:25:15.867990+0000 mgr.y (mgr.14152) 193 : cluster 0 pgmap v165: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:18.853 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping: /dev/vg_nvme/lv_4 2024-04-13T07:25:18.853 INFO:teuthology.orchestra.run.smithi203.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2024-04-13T07:25:18.853 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10+0 records in 2024-04-13T07:25:18.853 INFO:teuthology.orchestra.run.smithi203.stderr:10+0 records out 2024-04-13T07:25:18.853 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0137452 s, 763 MB/s 2024-04-13T07:25:18.853 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping successful for: 2024-04-13T07:25:19.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:18 smithi203 bash[23480]: cluster 2024-04-13T07:25:17.868527+0000 mgr.y (mgr.14152) 194 : cluster 0 pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:19.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:18 smithi089 bash[21353]: cluster 2024-04-13T07:25:17.868527+0000 mgr.y (mgr.14152) 194 : cluster 0 pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:19.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:18 smithi089 bash[31522]: cluster 2024-04-13T07:25:17.868527+0000 mgr.y (mgr.14152) 194 : cluster 0 pgmap v166: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:19.565 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi203:vg_nvme/lv_4 2024-04-13T07:25:21.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: cluster 2024-04-13T07:25:19.869053+0000 mgr.y (mgr.14152) 195 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:21.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: cephadm 2024-04-13T07:25:20.186811+0000 mgr.y (mgr.14152) 196 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:25:21.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.198612+0000 mon.a (mon.0) 430 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.211366+0000 mon.a (mon.0) 431 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.214142+0000 mon.a (mon.0) 432 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.215930+0000 mon.a (mon.0) 433 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.217580+0000 mon.a (mon.0) 434 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.219211+0000 mon.a (mon.0) 435 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: cephadm 2024-04-13T07:25:20.220421+0000 mgr.y (mgr.14152) 197 : cephadm 1 Adjusting osd_memory_target on smithi089 to 4060M 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.228849+0000 mon.a (mon.0) 436 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.231596+0000 mon.a (mon.0) 437 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.233350+0000 mon.a (mon.0) 438 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:21.521 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:21 smithi203 bash[23480]: audit 2024-04-13T07:25:20.243098+0000 mon.a (mon.0) 439 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: cluster 2024-04-13T07:25:19.869053+0000 mgr.y (mgr.14152) 195 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: cephadm 2024-04-13T07:25:20.186811+0000 mgr.y (mgr.14152) 196 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.198612+0000 mon.a (mon.0) 430 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.211366+0000 mon.a (mon.0) 431 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.214142+0000 mon.a (mon.0) 432 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.215930+0000 mon.a (mon.0) 433 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.217580+0000 mon.a (mon.0) 434 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.219211+0000 mon.a (mon.0) 435 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: cephadm 2024-04-13T07:25:20.220421+0000 mgr.y (mgr.14152) 197 : cephadm 1 Adjusting osd_memory_target on smithi089 to 4060M 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.228849+0000 mon.a (mon.0) 436 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.231596+0000 mon.a (mon.0) 437 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.233350+0000 mon.a (mon.0) 438 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:21.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[21353]: audit 2024-04-13T07:25:20.243098+0000 mon.a (mon.0) 439 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: cluster 2024-04-13T07:25:19.869053+0000 mgr.y (mgr.14152) 195 : cluster 0 pgmap v167: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: cephadm 2024-04-13T07:25:20.186811+0000 mgr.y (mgr.14152) 196 : cephadm 1 Detected new or changed devices on smithi089 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.198612+0000 mon.a (mon.0) 430 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.211366+0000 mon.a (mon.0) 431 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.214142+0000 mon.a (mon.0) 432 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.215930+0000 mon.a (mon.0) 433 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.217580+0000 mon.a (mon.0) 434 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.219211+0000 mon.a (mon.0) 435 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: cephadm 2024-04-13T07:25:20.220421+0000 mgr.y (mgr.14152) 197 : cephadm 1 Adjusting osd_memory_target on smithi089 to 4060M 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.228849+0000 mon.a (mon.0) 436 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.231596+0000 mon.a (mon.0) 437 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:21.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.233350+0000 mon.a (mon.0) 438 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:21.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:21 smithi089 bash[31522]: audit 2024-04-13T07:25:20.243098+0000 mon.a (mon.0) 439 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:23.522 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:23 smithi203 bash[23480]: cluster 2024-04-13T07:25:21.869567+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:23.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:23 smithi089 bash[21353]: cluster 2024-04-13T07:25:21.869567+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:23.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:23 smithi089 bash[31522]: cluster 2024-04-13T07:25:21.869567+0000 mgr.y (mgr.14152) 198 : cluster 0 pgmap v168: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:24.232 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:25:25.405 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:25 smithi203 bash[23480]: cluster 2024-04-13T07:25:23.870171+0000 mgr.y (mgr.14152) 199 : cluster 0 pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:25.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:25 smithi089 bash[21353]: cluster 2024-04-13T07:25:23.870171+0000 mgr.y (mgr.14152) 199 : cluster 0 pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:25.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:25 smithi089 bash[31522]: cluster 2024-04-13T07:25:23.870171+0000 mgr.y (mgr.14152) 199 : cluster 0 pgmap v169: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:27.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:27 smithi203 bash[23480]: cluster 2024-04-13T07:25:25.870660+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:27.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:27 smithi203 bash[23480]: audit 2024-04-13T07:25:27.019711+0000 mon.a (mon.0) 440 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:25:27.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:27 smithi203 bash[23480]: audit 2024-04-13T07:25:27.026792+0000 mon.a (mon.0) 441 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:25:27.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:27 smithi203 bash[23480]: audit 2024-04-13T07:25:27.028558+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:27.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:27 smithi089 bash[21353]: cluster 2024-04-13T07:25:25.870660+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:27.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:27 smithi089 bash[21353]: audit 2024-04-13T07:25:27.019711+0000 mon.a (mon.0) 440 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:25:27.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:27 smithi089 bash[21353]: audit 2024-04-13T07:25:27.026792+0000 mon.a (mon.0) 441 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:25:27.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:27 smithi089 bash[21353]: audit 2024-04-13T07:25:27.028558+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:27.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:27 smithi089 bash[31522]: cluster 2024-04-13T07:25:25.870660+0000 mgr.y (mgr.14152) 200 : cluster 0 pgmap v170: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:27.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:27 smithi089 bash[31522]: audit 2024-04-13T07:25:27.019711+0000 mon.a (mon.0) 440 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:25:27.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:27 smithi089 bash[31522]: audit 2024-04-13T07:25:27.026792+0000 mon.a (mon.0) 441 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:25:27.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:27 smithi089 bash[31522]: audit 2024-04-13T07:25:27.028558+0000 mon.a (mon.0) 442 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:28.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:28 smithi203 bash[23480]: audit 2024-04-13T07:25:27.012478+0000 mgr.y (mgr.14152) 201 : audit 0 from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:25:28.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:28 smithi089 bash[21353]: audit 2024-04-13T07:25:27.012478+0000 mgr.y (mgr.14152) 201 : audit 0 from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:25:28.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:28 smithi089 bash[31522]: audit 2024-04-13T07:25:27.012478+0000 mgr.y (mgr.14152) 201 : audit 0 from='client.24160 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:25:29.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:29 smithi203 bash[23480]: cluster 2024-04-13T07:25:27.871209+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:29.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:29 smithi089 bash[21353]: cluster 2024-04-13T07:25:27.871209+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:29.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:29 smithi089 bash[31522]: cluster 2024-04-13T07:25:27.871209+0000 mgr.y (mgr.14152) 202 : cluster 0 pgmap v171: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:31.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:31 smithi203 bash[23480]: cluster 2024-04-13T07:25:29.871714+0000 mgr.y (mgr.14152) 203 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:31.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:31 smithi089 bash[21353]: cluster 2024-04-13T07:25:29.871714+0000 mgr.y (mgr.14152) 203 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:31.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:31 smithi089 bash[31522]: cluster 2024-04-13T07:25:29.871714+0000 mgr.y (mgr.14152) 203 : cluster 0 pgmap v172: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:33.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:33 smithi203 bash[23480]: cluster 2024-04-13T07:25:31.872209+0000 mgr.y (mgr.14152) 204 : cluster 0 pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:33.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:33 smithi089 bash[31522]: cluster 2024-04-13T07:25:31.872209+0000 mgr.y (mgr.14152) 204 : cluster 0 pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:33.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:33 smithi089 bash[21353]: cluster 2024-04-13T07:25:31.872209+0000 mgr.y (mgr.14152) 204 : cluster 0 pgmap v173: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:35.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:35 smithi203 bash[23480]: cluster 2024-04-13T07:25:33.872741+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:35.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:35 smithi203 bash[23480]: audit 2024-04-13T07:25:35.080244+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.203:0/3630118495' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]: dispatch 2024-04-13T07:25:35.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:35 smithi203 bash[23480]: audit 2024-04-13T07:25:35.081825+0000 mon.a (mon.0) 443 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]: dispatch 2024-04-13T07:25:35.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:35 smithi203 bash[23480]: audit 2024-04-13T07:25:35.090636+0000 mon.a (mon.0) 444 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]': finished 2024-04-13T07:25:35.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:35 smithi203 bash[23480]: cluster 2024-04-13T07:25:35.090839+0000 mon.a (mon.0) 445 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-04-13T07:25:35.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:35 smithi203 bash[23480]: audit 2024-04-13T07:25:35.091131+0000 mon.a (mon.0) 446 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:25:35.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[21353]: cluster 2024-04-13T07:25:33.872741+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:35.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[21353]: audit 2024-04-13T07:25:35.080244+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.203:0/3630118495' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]: dispatch 2024-04-13T07:25:35.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[21353]: audit 2024-04-13T07:25:35.081825+0000 mon.a (mon.0) 443 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]: dispatch 2024-04-13T07:25:35.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[21353]: audit 2024-04-13T07:25:35.090636+0000 mon.a (mon.0) 444 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]': finished 2024-04-13T07:25:35.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[21353]: cluster 2024-04-13T07:25:35.090839+0000 mon.a (mon.0) 445 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-04-13T07:25:35.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[21353]: audit 2024-04-13T07:25:35.091131+0000 mon.a (mon.0) 446 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:25:35.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[31522]: cluster 2024-04-13T07:25:33.872741+0000 mgr.y (mgr.14152) 205 : cluster 0 pgmap v174: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:35.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[31522]: audit 2024-04-13T07:25:35.080244+0000 mon.b (mon.1) 9 : audit 1 from='client.? 172.21.15.203:0/3630118495' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]: dispatch 2024-04-13T07:25:35.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[31522]: audit 2024-04-13T07:25:35.081825+0000 mon.a (mon.0) 443 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]: dispatch 2024-04-13T07:25:35.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[31522]: audit 2024-04-13T07:25:35.090636+0000 mon.a (mon.0) 444 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "c1befaa2-c9f5-4023-9909-0577509ce44d"}]': finished 2024-04-13T07:25:35.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[31522]: cluster 2024-04-13T07:25:35.090839+0000 mon.a (mon.0) 445 : cluster 0 osdmap e28: 5 total, 4 up, 5 in 2024-04-13T07:25:35.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:35 smithi089 bash[31522]: audit 2024-04-13T07:25:35.091131+0000 mon.a (mon.0) 446 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:25:36.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:36 smithi203 bash[23480]: audit 2024-04-13T07:25:35.838156+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.203:0/1854771587' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:25:36.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:36 smithi089 bash[21353]: audit 2024-04-13T07:25:35.838156+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.203:0/1854771587' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:25:36.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:36 smithi089 bash[31522]: audit 2024-04-13T07:25:35.838156+0000 mon.b (mon.1) 10 : audit 0 from='client.? 172.21.15.203:0/1854771587' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:25:37.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:37 smithi203 bash[23480]: cluster 2024-04-13T07:25:35.873239+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:37.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:37 smithi089 bash[21353]: cluster 2024-04-13T07:25:35.873239+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:37.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:37 smithi089 bash[31522]: cluster 2024-04-13T07:25:35.873239+0000 mgr.y (mgr.14152) 206 : cluster 0 pgmap v176: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:39.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:39 smithi203 bash[23480]: cluster 2024-04-13T07:25:37.873764+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:39.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:39 smithi089 bash[31522]: cluster 2024-04-13T07:25:37.873764+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:39.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:39 smithi089 bash[21353]: cluster 2024-04-13T07:25:37.873764+0000 mgr.y (mgr.14152) 207 : cluster 0 pgmap v177: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:41.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:41 smithi203 bash[23480]: cluster 2024-04-13T07:25:39.874341+0000 mgr.y (mgr.14152) 208 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:41.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:41 smithi089 bash[21353]: cluster 2024-04-13T07:25:39.874341+0000 mgr.y (mgr.14152) 208 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:41.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:41 smithi089 bash[31522]: cluster 2024-04-13T07:25:39.874341+0000 mgr.y (mgr.14152) 208 : cluster 0 pgmap v178: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:43.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:43 smithi203 bash[23480]: cluster 2024-04-13T07:25:41.874847+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:43.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:43 smithi089 bash[21353]: cluster 2024-04-13T07:25:41.874847+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:43.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:43 smithi089 bash[31522]: cluster 2024-04-13T07:25:41.874847+0000 mgr.y (mgr.14152) 209 : cluster 0 pgmap v179: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:45.512 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:45 smithi203 bash[23480]: cluster 2024-04-13T07:25:43.875403+0000 mgr.y (mgr.14152) 210 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:45.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:45 smithi089 bash[21353]: cluster 2024-04-13T07:25:43.875403+0000 mgr.y (mgr.14152) 210 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:45.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:45 smithi089 bash[31522]: cluster 2024-04-13T07:25:43.875403+0000 mgr.y (mgr.14152) 210 : cluster 0 pgmap v180: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:47.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:47 smithi203 bash[23480]: cluster 2024-04-13T07:25:45.875896+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:47.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:47 smithi089 bash[21353]: cluster 2024-04-13T07:25:45.875896+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:47.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:47 smithi089 bash[31522]: cluster 2024-04-13T07:25:45.875896+0000 mgr.y (mgr.14152) 211 : cluster 0 pgmap v181: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:48.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:48 smithi089 bash[21353]: audit 2024-04-13T07:25:47.373120+0000 mon.a (mon.0) 447 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:25:48.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:48 smithi089 bash[21353]: audit 2024-04-13T07:25:47.374760+0000 mon.a (mon.0) 448 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:48.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:48 smithi089 bash[21353]: cephadm 2024-04-13T07:25:47.376292+0000 mgr.y (mgr.14152) 212 : cephadm 1 Deploying daemon osd.4 on smithi203 2024-04-13T07:25:48.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:48 smithi089 bash[31522]: audit 2024-04-13T07:25:47.373120+0000 mon.a (mon.0) 447 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:25:48.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:48 smithi089 bash[31522]: audit 2024-04-13T07:25:47.374760+0000 mon.a (mon.0) 448 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:48.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:48 smithi089 bash[31522]: cephadm 2024-04-13T07:25:47.376292+0000 mgr.y (mgr.14152) 212 : cephadm 1 Deploying daemon osd.4 on smithi203 2024-04-13T07:25:48.553 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:48 smithi203 bash[23480]: audit 2024-04-13T07:25:47.373120+0000 mon.a (mon.0) 447 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:25:48.553 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:48 smithi203 bash[23480]: audit 2024-04-13T07:25:47.374760+0000 mon.a (mon.0) 448 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:48.553 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:48 smithi203 bash[23480]: cephadm 2024-04-13T07:25:47.376292+0000 mgr.y (mgr.14152) 212 : cephadm 1 Deploying daemon osd.4 on smithi203 2024-04-13T07:25:49.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:49 smithi089 bash[21353]: cluster 2024-04-13T07:25:47.876460+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:49.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:49 smithi089 bash[31522]: cluster 2024-04-13T07:25:47.876460+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:49.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:49 smithi203 bash[23480]: cluster 2024-04-13T07:25:47.876460+0000 mgr.y (mgr.14152) 213 : cluster 0 pgmap v182: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:51.415 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:25:51 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:51.416 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:51 smithi203 bash[23480]: cluster 2024-04-13T07:25:49.876993+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:51.416 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:51 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:51.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:51 smithi089 bash[21353]: cluster 2024-04-13T07:25:49.876993+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:51.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:51 smithi089 bash[31522]: cluster 2024-04-13T07:25:49.876993+0000 mgr.y (mgr.14152) 214 : cluster 0 pgmap v183: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:51.684 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:25:51 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:51.685 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:51 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:25:53.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:52 smithi203 bash[23480]: audit 2024-04-13T07:25:51.684331+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:53.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:52 smithi203 bash[23480]: audit 2024-04-13T07:25:51.692916+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:53.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:52 smithi203 bash[23480]: audit 2024-04-13T07:25:51.712950+0000 mon.a (mon.0) 451 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:53.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:52 smithi089 bash[21353]: audit 2024-04-13T07:25:51.684331+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:53.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:52 smithi089 bash[21353]: audit 2024-04-13T07:25:51.692916+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:53.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:52 smithi089 bash[21353]: audit 2024-04-13T07:25:51.712950+0000 mon.a (mon.0) 451 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:53.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:52 smithi089 bash[31522]: audit 2024-04-13T07:25:51.684331+0000 mon.a (mon.0) 449 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:53.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:52 smithi089 bash[31522]: audit 2024-04-13T07:25:51.692916+0000 mon.a (mon.0) 450 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:53.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:52 smithi089 bash[31522]: audit 2024-04-13T07:25:51.712950+0000 mon.a (mon.0) 451 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:25:54.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:53 smithi203 bash[23480]: cluster 2024-04-13T07:25:51.877501+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:54.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:53 smithi089 bash[21353]: cluster 2024-04-13T07:25:51.877501+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:54.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:53 smithi089 bash[31522]: cluster 2024-04-13T07:25:51.877501+0000 mgr.y (mgr.14152) 215 : cluster 0 pgmap v184: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:56.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:55 smithi203 bash[23480]: cluster 2024-04-13T07:25:53.877987+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:56.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:55 smithi089 bash[21353]: cluster 2024-04-13T07:25:53.877987+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:56.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:55 smithi089 bash[31522]: cluster 2024-04-13T07:25:53.877987+0000 mgr.y (mgr.14152) 216 : cluster 0 pgmap v185: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:58.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:57 smithi203 bash[23480]: cluster 2024-04-13T07:25:55.878554+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:58.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:57 smithi203 bash[23480]: audit 2024-04-13T07:25:56.859802+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:58.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:57 smithi203 bash[23480]: audit 2024-04-13T07:25:56.870239+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:58.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:57 smithi203 bash[23480]: audit 2024-04-13T07:25:56.872371+0000 mon.a (mon.0) 454 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:58.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:57 smithi203 bash[23480]: audit 2024-04-13T07:25:56.874255+0000 mon.a (mon.0) 455 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:58.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:57 smithi203 bash[23480]: audit 2024-04-13T07:25:56.885641+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:58.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[21353]: cluster 2024-04-13T07:25:55.878554+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:58.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[21353]: audit 2024-04-13T07:25:56.859802+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:58.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[21353]: audit 2024-04-13T07:25:56.870239+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[21353]: audit 2024-04-13T07:25:56.872371+0000 mon.a (mon.0) 454 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[21353]: audit 2024-04-13T07:25:56.874255+0000 mon.a (mon.0) 455 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[21353]: audit 2024-04-13T07:25:56.885641+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[31522]: cluster 2024-04-13T07:25:55.878554+0000 mgr.y (mgr.14152) 217 : cluster 0 pgmap v186: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[31522]: audit 2024-04-13T07:25:56.859802+0000 mon.a (mon.0) 452 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[31522]: audit 2024-04-13T07:25:56.870239+0000 mon.a (mon.0) 453 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[31522]: audit 2024-04-13T07:25:56.872371+0000 mon.a (mon.0) 454 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[31522]: audit 2024-04-13T07:25:56.874255+0000 mon.a (mon.0) 455 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:25:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:57 smithi089 bash[31522]: audit 2024-04-13T07:25:56.885641+0000 mon.a (mon.0) 456 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:25:59.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:58 smithi203 bash[23480]: audit 2024-04-13T07:25:58.107844+0000 mon.b (mon.1) 11 : audit 1 from='osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:25:59.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:58 smithi203 bash[23480]: audit 2024-04-13T07:25:58.109039+0000 mon.a (mon.0) 457 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:25:59.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:58 smithi089 bash[21353]: audit 2024-04-13T07:25:58.107844+0000 mon.b (mon.1) 11 : audit 1 from='osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:25:59.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:58 smithi089 bash[21353]: audit 2024-04-13T07:25:58.109039+0000 mon.a (mon.0) 457 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:25:59.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:58 smithi089 bash[31522]: audit 2024-04-13T07:25:58.107844+0000 mon.b (mon.1) 11 : audit 1 from='osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:25:59.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:58 smithi089 bash[31522]: audit 2024-04-13T07:25:58.109039+0000 mon.a (mon.0) 457 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:26:00.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:59 smithi203 bash[23480]: cluster 2024-04-13T07:25:57.879098+0000 mgr.y (mgr.14152) 218 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:00.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:59 smithi203 bash[23480]: audit 2024-04-13T07:25:58.879863+0000 mon.a (mon.0) 458 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-13T07:26:00.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:59 smithi203 bash[23480]: cluster 2024-04-13T07:25:58.880201+0000 mon.a (mon.0) 459 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-04-13T07:26:00.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:59 smithi203 bash[23480]: audit 2024-04-13T07:25:58.880690+0000 mon.a (mon.0) 460 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:00.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:59 smithi203 bash[23480]: audit 2024-04-13T07:25:58.881333+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:00.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:25:59 smithi203 bash[23480]: audit 2024-04-13T07:25:58.882637+0000 mon.a (mon.0) 461 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:00.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[31522]: cluster 2024-04-13T07:25:57.879098+0000 mgr.y (mgr.14152) 218 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:00.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[31522]: audit 2024-04-13T07:25:58.879863+0000 mon.a (mon.0) 458 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-13T07:26:00.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[31522]: cluster 2024-04-13T07:25:58.880201+0000 mon.a (mon.0) 459 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-04-13T07:26:00.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[31522]: audit 2024-04-13T07:25:58.880690+0000 mon.a (mon.0) 460 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:00.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[31522]: audit 2024-04-13T07:25:58.881333+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:00.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[31522]: audit 2024-04-13T07:25:58.882637+0000 mon.a (mon.0) 461 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:00.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[21353]: cluster 2024-04-13T07:25:57.879098+0000 mgr.y (mgr.14152) 218 : cluster 0 pgmap v187: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:00.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[21353]: audit 2024-04-13T07:25:58.879863+0000 mon.a (mon.0) 458 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-13T07:26:00.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[21353]: cluster 2024-04-13T07:25:58.880201+0000 mon.a (mon.0) 459 : cluster 0 osdmap e29: 5 total, 4 up, 5 in 2024-04-13T07:26:00.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[21353]: audit 2024-04-13T07:25:58.880690+0000 mon.a (mon.0) 460 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:00.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[21353]: audit 2024-04-13T07:25:58.881333+0000 mon.b (mon.1) 12 : audit 1 from='osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:00.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:25:59 smithi089 bash[21353]: audit 2024-04-13T07:25:58.882637+0000 mon.a (mon.0) 461 : audit 1 from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:00.447 INFO:teuthology.orchestra.run.smithi203.stdout:Created osd(s) 4 on host 'smithi203' 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: audit 2024-04-13T07:25:59.879101+0000 mon.a (mon.0) 462 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: cluster 2024-04-13T07:25:59.879301+0000 mon.a (mon.0) 463 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: audit 2024-04-13T07:25:59.879669+0000 mon.a (mon.0) 464 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: cluster 2024-04-13T07:25:59.879715+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: audit 2024-04-13T07:25:59.887925+0000 mon.a (mon.0) 465 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: audit 2024-04-13T07:26:00.416355+0000 mon.a (mon.0) 466 : audit 1 from='osd.4 ' entity='osd.4' 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: audit 2024-04-13T07:26:00.426773+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: audit 2024-04-13T07:26:00.431147+0000 mon.a (mon.0) 468 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:01.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:00 smithi203 bash[23480]: audit 2024-04-13T07:26:00.436606+0000 mon.a (mon.0) 469 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:01.142 DEBUG:teuthology.orchestra.run.smithi203:osd.4> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.4.service 2024-04-13T07:26:01.146 INFO:tasks.cephadm:Deploying osd.5 on smithi203 with /dev/vg_nvme/lv_3... 2024-04-13T07:26:01.146 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2024-04-13T07:26:01.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: audit 2024-04-13T07:25:59.879101+0000 mon.a (mon.0) 462 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: cluster 2024-04-13T07:25:59.879301+0000 mon.a (mon.0) 463 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: audit 2024-04-13T07:25:59.879669+0000 mon.a (mon.0) 464 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: cluster 2024-04-13T07:25:59.879715+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: audit 2024-04-13T07:25:59.887925+0000 mon.a (mon.0) 465 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: audit 2024-04-13T07:26:00.416355+0000 mon.a (mon.0) 466 : audit 1 from='osd.4 ' entity='osd.4' 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: audit 2024-04-13T07:26:00.426773+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: audit 2024-04-13T07:26:00.431147+0000 mon.a (mon.0) 468 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[21353]: audit 2024-04-13T07:26:00.436606+0000 mon.a (mon.0) 469 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: audit 2024-04-13T07:25:59.879101+0000 mon.a (mon.0) 462 : audit 1 from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: cluster 2024-04-13T07:25:59.879301+0000 mon.a (mon.0) 463 : cluster 0 osdmap e30: 5 total, 4 up, 5 in 2024-04-13T07:26:01.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: audit 2024-04-13T07:25:59.879669+0000 mon.a (mon.0) 464 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:01.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: cluster 2024-04-13T07:25:59.879715+0000 mgr.y (mgr.14152) 219 : cluster 0 pgmap v190: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:01.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: audit 2024-04-13T07:25:59.887925+0000 mon.a (mon.0) 465 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:01.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: audit 2024-04-13T07:26:00.416355+0000 mon.a (mon.0) 466 : audit 1 from='osd.4 ' entity='osd.4' 2024-04-13T07:26:01.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: audit 2024-04-13T07:26:00.426773+0000 mon.a (mon.0) 467 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:01.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: audit 2024-04-13T07:26:00.431147+0000 mon.a (mon.0) 468 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:01.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:00 smithi089 bash[31522]: audit 2024-04-13T07:26:00.436606+0000 mon.a (mon.0) 469 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:02.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:01 smithi203 bash[23480]: cluster 2024-04-13T07:25:59.130631+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:26:02.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:01 smithi203 bash[23480]: cluster 2024-04-13T07:25:59.130693+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:26:02.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:01 smithi203 bash[23480]: cluster 2024-04-13T07:26:00.895996+0000 mon.a (mon.0) 470 : cluster 1 osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878] boot 2024-04-13T07:26:02.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:01 smithi203 bash[23480]: cluster 2024-04-13T07:26:00.896120+0000 mon.a (mon.0) 471 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-04-13T07:26:02.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:01 smithi203 bash[23480]: audit 2024-04-13T07:26:00.896465+0000 mon.a (mon.0) 472 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:02.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[21353]: cluster 2024-04-13T07:25:59.130631+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:26:02.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[21353]: cluster 2024-04-13T07:25:59.130693+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:26:02.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[21353]: cluster 2024-04-13T07:26:00.895996+0000 mon.a (mon.0) 470 : cluster 1 osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878] boot 2024-04-13T07:26:02.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[21353]: cluster 2024-04-13T07:26:00.896120+0000 mon.a (mon.0) 471 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-04-13T07:26:02.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[21353]: audit 2024-04-13T07:26:00.896465+0000 mon.a (mon.0) 472 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:02.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[31522]: cluster 2024-04-13T07:25:59.130631+0000 osd.4 (osd.4) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:26:02.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[31522]: cluster 2024-04-13T07:25:59.130693+0000 osd.4 (osd.4) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:26:02.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[31522]: cluster 2024-04-13T07:26:00.895996+0000 mon.a (mon.0) 470 : cluster 1 osd.4 [v2:172.21.15.203:6800/3982510878,v1:172.21.15.203:6801/3982510878] boot 2024-04-13T07:26:02.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[31522]: cluster 2024-04-13T07:26:00.896120+0000 mon.a (mon.0) 471 : cluster 0 osdmap e31: 5 total, 5 up, 5 in 2024-04-13T07:26:02.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:01 smithi089 bash[31522]: audit 2024-04-13T07:26:00.896465+0000 mon.a (mon.0) 472 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:26:03.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:02 smithi203 bash[23480]: cluster 2024-04-13T07:26:01.880259+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:03.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:02 smithi203 bash[23480]: cluster 2024-04-13T07:26:01.901308+0000 mon.a (mon.0) 473 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-04-13T07:26:03.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:02 smithi089 bash[21353]: cluster 2024-04-13T07:26:01.880259+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:03.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:02 smithi089 bash[21353]: cluster 2024-04-13T07:26:01.901308+0000 mon.a (mon.0) 473 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-04-13T07:26:03.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:02 smithi089 bash[31522]: cluster 2024-04-13T07:26:01.880259+0000 mgr.y (mgr.14152) 220 : cluster 0 pgmap v192: 1 pgs: 1 active+clean; 577 KiB data, 108 MiB used, 358 GiB / 358 GiB avail 2024-04-13T07:26:03.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:02 smithi089 bash[31522]: cluster 2024-04-13T07:26:01.901308+0000 mon.a (mon.0) 473 : cluster 0 osdmap e32: 5 total, 5 up, 5 in 2024-04-13T07:26:04.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:03 smithi203 bash[23480]: cluster 2024-04-13T07:26:02.902442+0000 mon.a (mon.0) 474 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-04-13T07:26:04.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:03 smithi089 bash[21353]: cluster 2024-04-13T07:26:02.902442+0000 mon.a (mon.0) 474 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-04-13T07:26:04.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:03 smithi089 bash[31522]: cluster 2024-04-13T07:26:02.902442+0000 mon.a (mon.0) 474 : cluster 0 osdmap e33: 5 total, 5 up, 5 in 2024-04-13T07:26:05.251 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:26:05.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:04 smithi203 bash[23480]: cluster 2024-04-13T07:26:03.880851+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:05.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:04 smithi203 bash[23480]: cluster 2024-04-13T07:26:03.924745+0000 mon.a (mon.0) 475 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-04-13T07:26:05.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:04 smithi089 bash[21353]: cluster 2024-04-13T07:26:03.880851+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:05.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:04 smithi089 bash[21353]: cluster 2024-04-13T07:26:03.924745+0000 mon.a (mon.0) 475 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-04-13T07:26:05.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:04 smithi089 bash[31522]: cluster 2024-04-13T07:26:03.880851+0000 mgr.y (mgr.14152) 221 : cluster 0 pgmap v195: 1 pgs: 1 active+clean; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:05.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:04 smithi089 bash[31522]: cluster 2024-04-13T07:26:03.924745+0000 mon.a (mon.0) 475 : cluster 0 osdmap e34: 5 total, 5 up, 5 in 2024-04-13T07:26:06.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:05 smithi203 bash[23480]: cluster 2024-04-13T07:26:04.931162+0000 mon.a (mon.0) 476 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-04-13T07:26:06.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:05 smithi089 bash[21353]: cluster 2024-04-13T07:26:04.931162+0000 mon.a (mon.0) 476 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-04-13T07:26:06.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:05 smithi089 bash[31522]: cluster 2024-04-13T07:26:04.931162+0000 mon.a (mon.0) 476 : cluster 0 osdmap e35: 5 total, 5 up, 5 in 2024-04-13T07:26:07.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:06 smithi203 bash[23480]: cluster 2024-04-13T07:26:05.881514+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v198: 1 pgs: 1 peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-13T07:26:07.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:06 smithi089 bash[21353]: cluster 2024-04-13T07:26:05.881514+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v198: 1 pgs: 1 peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-13T07:26:07.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:06 smithi089 bash[31522]: cluster 2024-04-13T07:26:05.881514+0000 mgr.y (mgr.14152) 222 : cluster 0 pgmap v198: 1 pgs: 1 peering; 577 KiB data, 139 MiB used, 447 GiB / 447 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-13T07:26:09.204 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:08 smithi203 bash[23480]: cluster 2024-04-13T07:26:07.882141+0000 mgr.y (mgr.14152) 223 : cluster 0 pgmap v199: 1 pgs: 1 peering; 577 KiB data, 140 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:26:09.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:08 smithi089 bash[21353]: cluster 2024-04-13T07:26:07.882141+0000 mgr.y (mgr.14152) 223 : cluster 0 pgmap v199: 1 pgs: 1 peering; 577 KiB data, 140 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:26:09.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:08 smithi089 bash[31522]: cluster 2024-04-13T07:26:07.882141+0000 mgr.y (mgr.14152) 223 : cluster 0 pgmap v199: 1 pgs: 1 peering; 577 KiB data, 140 MiB used, 447 GiB / 447 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:26:09.907 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping: /dev/vg_nvme/lv_3 2024-04-13T07:26:09.907 INFO:teuthology.orchestra.run.smithi203.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2024-04-13T07:26:09.907 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10+0 records in 2024-04-13T07:26:09.907 INFO:teuthology.orchestra.run.smithi203.stderr:10+0 records out 2024-04-13T07:26:09.907 INFO:teuthology.orchestra.run.smithi203.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0130839 s, 801 MB/s 2024-04-13T07:26:09.907 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 2024-04-13T07:26:09.907 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping successful for: 2024-04-13T07:26:10.856 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi203:vg_nvme/lv_3 2024-04-13T07:26:11.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:10 smithi203 bash[23480]: cluster 2024-04-13T07:26:09.882760+0000 mgr.y (mgr.14152) 224 : cluster 0 pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-13T07:26:11.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:10 smithi089 bash[21353]: cluster 2024-04-13T07:26:09.882760+0000 mgr.y (mgr.14152) 224 : cluster 0 pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-13T07:26:11.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:10 smithi089 bash[31522]: cluster 2024-04-13T07:26:09.882760+0000 mgr.y (mgr.14152) 224 : cluster 0 pgmap v200: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 83 KiB/s, 0 objects/s recovering 2024-04-13T07:26:12.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: cephadm 2024-04-13T07:26:11.235684+0000 mgr.y (mgr.14152) 225 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:26:12.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: audit 2024-04-13T07:26:11.244771+0000 mon.a (mon.0) 477 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: audit 2024-04-13T07:26:11.254382+0000 mon.a (mon.0) 478 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: audit 2024-04-13T07:26:11.256615+0000 mon.a (mon.0) 479 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:26:12.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: cephadm 2024-04-13T07:26:11.258107+0000 mgr.y (mgr.14152) 226 : cephadm 1 Adjusting osd_memory_target on smithi203 to 17265M 2024-04-13T07:26:12.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: audit 2024-04-13T07:26:11.266839+0000 mon.a (mon.0) 480 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: audit 2024-04-13T07:26:11.269018+0000 mon.a (mon.0) 481 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:12.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: audit 2024-04-13T07:26:11.270615+0000 mon.a (mon.0) 482 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:26:12.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:12 smithi203 bash[23480]: audit 2024-04-13T07:26:11.280460+0000 mon.a (mon.0) 483 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: cephadm 2024-04-13T07:26:11.235684+0000 mgr.y (mgr.14152) 225 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:26:12.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: audit 2024-04-13T07:26:11.244771+0000 mon.a (mon.0) 477 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: audit 2024-04-13T07:26:11.254382+0000 mon.a (mon.0) 478 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: audit 2024-04-13T07:26:11.256615+0000 mon.a (mon.0) 479 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: cephadm 2024-04-13T07:26:11.258107+0000 mgr.y (mgr.14152) 226 : cephadm 1 Adjusting osd_memory_target on smithi203 to 17265M 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: audit 2024-04-13T07:26:11.266839+0000 mon.a (mon.0) 480 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: audit 2024-04-13T07:26:11.269018+0000 mon.a (mon.0) 481 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: audit 2024-04-13T07:26:11.270615+0000 mon.a (mon.0) 482 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[21353]: audit 2024-04-13T07:26:11.280460+0000 mon.a (mon.0) 483 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: cephadm 2024-04-13T07:26:11.235684+0000 mgr.y (mgr.14152) 225 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: audit 2024-04-13T07:26:11.244771+0000 mon.a (mon.0) 477 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: audit 2024-04-13T07:26:11.254382+0000 mon.a (mon.0) 478 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: audit 2024-04-13T07:26:11.256615+0000 mon.a (mon.0) 479 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:26:12.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: cephadm 2024-04-13T07:26:11.258107+0000 mgr.y (mgr.14152) 226 : cephadm 1 Adjusting osd_memory_target on smithi203 to 17265M 2024-04-13T07:26:12.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: audit 2024-04-13T07:26:11.266839+0000 mon.a (mon.0) 480 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:12.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: audit 2024-04-13T07:26:11.269018+0000 mon.a (mon.0) 481 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:12.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: audit 2024-04-13T07:26:11.270615+0000 mon.a (mon.0) 482 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:26:12.554 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:12 smithi089 bash[31522]: audit 2024-04-13T07:26:11.280460+0000 mon.a (mon.0) 483 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:13.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:13 smithi089 bash[21353]: cluster 2024-04-13T07:26:11.883335+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-13T07:26:13.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:13 smithi089 bash[31522]: cluster 2024-04-13T07:26:11.883335+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-13T07:26:13.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:13 smithi203 bash[23480]: cluster 2024-04-13T07:26:11.883335+0000 mgr.y (mgr.14152) 227 : cluster 0 pgmap v201: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2024-04-13T07:26:15.534 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:26:15.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:15 smithi089 bash[21353]: cluster 2024-04-13T07:26:13.883934+0000 mgr.y (mgr.14152) 228 : cluster 0 pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-13T07:26:15.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:15 smithi089 bash[31522]: cluster 2024-04-13T07:26:13.883934+0000 mgr.y (mgr.14152) 228 : cluster 0 pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-13T07:26:15.555 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:15 smithi203 bash[23480]: cluster 2024-04-13T07:26:13.883934+0000 mgr.y (mgr.14152) 228 : cluster 0 pgmap v202: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-13T07:26:17.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:17 smithi089 bash[21353]: cluster 2024-04-13T07:26:15.884509+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:17.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:17 smithi089 bash[31522]: cluster 2024-04-13T07:26:15.884509+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:17.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:17 smithi203 bash[23480]: cluster 2024-04-13T07:26:15.884509+0000 mgr.y (mgr.14152) 229 : cluster 0 pgmap v203: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:19.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[21353]: cluster 2024-04-13T07:26:17.885080+0000 mgr.y (mgr.14152) 230 : cluster 0 pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:19.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[21353]: audit 2024-04-13T07:26:18.415511+0000 mgr.y (mgr.14152) 231 : audit 0 from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:26:19.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[21353]: audit 2024-04-13T07:26:18.422477+0000 mon.a (mon.0) 484 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:26:19.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[21353]: audit 2024-04-13T07:26:18.429735+0000 mon.a (mon.0) 485 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:26:19.553 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[21353]: audit 2024-04-13T07:26:18.431387+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:19.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[31522]: cluster 2024-04-13T07:26:17.885080+0000 mgr.y (mgr.14152) 230 : cluster 0 pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:19.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[31522]: audit 2024-04-13T07:26:18.415511+0000 mgr.y (mgr.14152) 231 : audit 0 from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:26:19.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[31522]: audit 2024-04-13T07:26:18.422477+0000 mon.a (mon.0) 484 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:26:19.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[31522]: audit 2024-04-13T07:26:18.429735+0000 mon.a (mon.0) 485 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:26:19.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:19 smithi089 bash[31522]: audit 2024-04-13T07:26:18.431387+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:19.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:19 smithi203 bash[23480]: cluster 2024-04-13T07:26:17.885080+0000 mgr.y (mgr.14152) 230 : cluster 0 pgmap v204: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:19.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:19 smithi203 bash[23480]: audit 2024-04-13T07:26:18.415511+0000 mgr.y (mgr.14152) 231 : audit 0 from='client.24187 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:26:19.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:19 smithi203 bash[23480]: audit 2024-04-13T07:26:18.422477+0000 mon.a (mon.0) 484 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:26:19.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:19 smithi203 bash[23480]: audit 2024-04-13T07:26:18.429735+0000 mon.a (mon.0) 485 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:26:19.770 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:19 smithi203 bash[23480]: audit 2024-04-13T07:26:18.431387+0000 mon.a (mon.0) 486 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:21.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:21 smithi089 bash[21353]: cluster 2024-04-13T07:26:19.885728+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:21.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:21 smithi089 bash[31522]: cluster 2024-04-13T07:26:19.885728+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:21.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:21 smithi203 bash[23480]: cluster 2024-04-13T07:26:19.885728+0000 mgr.y (mgr.14152) 232 : cluster 0 pgmap v205: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:23.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:23 smithi089 bash[21353]: cluster 2024-04-13T07:26:21.886288+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:23.645 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:23 smithi089 bash[31522]: cluster 2024-04-13T07:26:21.886288+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:23.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:23 smithi203 bash[23480]: cluster 2024-04-13T07:26:21.886288+0000 mgr.y (mgr.14152) 233 : cluster 0 pgmap v206: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:25.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:25 smithi203 bash[23480]: cluster 2024-04-13T07:26:23.886851+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:25.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:25 smithi089 bash[21353]: cluster 2024-04-13T07:26:23.886851+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:25.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:25 smithi089 bash[31522]: cluster 2024-04-13T07:26:23.886851+0000 mgr.y (mgr.14152) 234 : cluster 0 pgmap v207: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:27.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:27 smithi203 bash[23480]: cluster 2024-04-13T07:26:25.887396+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:27.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:27 smithi203 bash[23480]: audit 2024-04-13T07:26:26.731478+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.203:0/3005109039' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]: dispatch 2024-04-13T07:26:27.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:27 smithi203 bash[23480]: audit 2024-04-13T07:26:26.732716+0000 mon.a (mon.0) 487 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]: dispatch 2024-04-13T07:26:27.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:27 smithi203 bash[23480]: audit 2024-04-13T07:26:26.741775+0000 mon.a (mon.0) 488 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]': finished 2024-04-13T07:26:27.770 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:27 smithi203 bash[23480]: cluster 2024-04-13T07:26:26.741924+0000 mon.a (mon.0) 489 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-04-13T07:26:27.770 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:27 smithi203 bash[23480]: audit 2024-04-13T07:26:26.742213+0000 mon.a (mon.0) 490 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:27.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[21353]: cluster 2024-04-13T07:26:25.887396+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:27.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[21353]: audit 2024-04-13T07:26:26.731478+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.203:0/3005109039' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]: dispatch 2024-04-13T07:26:27.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[21353]: audit 2024-04-13T07:26:26.732716+0000 mon.a (mon.0) 487 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]: dispatch 2024-04-13T07:26:27.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[21353]: audit 2024-04-13T07:26:26.741775+0000 mon.a (mon.0) 488 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]': finished 2024-04-13T07:26:27.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[21353]: cluster 2024-04-13T07:26:26.741924+0000 mon.a (mon.0) 489 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-04-13T07:26:27.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[21353]: audit 2024-04-13T07:26:26.742213+0000 mon.a (mon.0) 490 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:27.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[31522]: cluster 2024-04-13T07:26:25.887396+0000 mgr.y (mgr.14152) 235 : cluster 0 pgmap v208: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:27.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[31522]: audit 2024-04-13T07:26:26.731478+0000 mon.b (mon.1) 13 : audit 1 from='client.? 172.21.15.203:0/3005109039' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]: dispatch 2024-04-13T07:26:27.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[31522]: audit 2024-04-13T07:26:26.732716+0000 mon.a (mon.0) 487 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]: dispatch 2024-04-13T07:26:27.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[31522]: audit 2024-04-13T07:26:26.741775+0000 mon.a (mon.0) 488 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "b45987d4-fc0f-45ad-9f0d-49240d898bfc"}]': finished 2024-04-13T07:26:27.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[31522]: cluster 2024-04-13T07:26:26.741924+0000 mon.a (mon.0) 489 : cluster 0 osdmap e36: 6 total, 5 up, 6 in 2024-04-13T07:26:27.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:27 smithi089 bash[31522]: audit 2024-04-13T07:26:26.742213+0000 mon.a (mon.0) 490 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:28.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:28 smithi203 bash[23480]: audit 2024-04-13T07:26:27.512089+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.203:0/4154627999' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:26:28.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:28 smithi089 bash[21353]: audit 2024-04-13T07:26:27.512089+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.203:0/4154627999' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:26:28.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:28 smithi089 bash[31522]: audit 2024-04-13T07:26:27.512089+0000 mon.b (mon.1) 14 : audit 0 from='client.? 172.21.15.203:0/4154627999' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:26:29.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:29 smithi203 bash[23480]: cluster 2024-04-13T07:26:27.887904+0000 mgr.y (mgr.14152) 236 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:29.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:29 smithi089 bash[21353]: cluster 2024-04-13T07:26:27.887904+0000 mgr.y (mgr.14152) 236 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:29.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:29 smithi089 bash[31522]: cluster 2024-04-13T07:26:27.887904+0000 mgr.y (mgr.14152) 236 : cluster 0 pgmap v210: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:31.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:31 smithi203 bash[23480]: cluster 2024-04-13T07:26:29.888453+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:31.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:31 smithi089 bash[21353]: cluster 2024-04-13T07:26:29.888453+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:31.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:31 smithi089 bash[31522]: cluster 2024-04-13T07:26:29.888453+0000 mgr.y (mgr.14152) 237 : cluster 0 pgmap v211: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:33.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:33 smithi203 bash[23480]: cluster 2024-04-13T07:26:31.889000+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:33.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:33 smithi089 bash[21353]: cluster 2024-04-13T07:26:31.889000+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:33.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:33 smithi089 bash[31522]: cluster 2024-04-13T07:26:31.889000+0000 mgr.y (mgr.14152) 238 : cluster 0 pgmap v212: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:35.662 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:35 smithi203 bash[23480]: cluster 2024-04-13T07:26:33.889506+0000 mgr.y (mgr.14152) 239 : cluster 0 pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:35.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:35 smithi089 bash[21353]: cluster 2024-04-13T07:26:33.889506+0000 mgr.y (mgr.14152) 239 : cluster 0 pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:35.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:35 smithi089 bash[31522]: cluster 2024-04-13T07:26:33.889506+0000 mgr.y (mgr.14152) 239 : cluster 0 pgmap v213: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:37.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:37 smithi203 bash[23480]: cluster 2024-04-13T07:26:35.890144+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:37.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:37 smithi089 bash[21353]: cluster 2024-04-13T07:26:35.890144+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:37.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:37 smithi089 bash[31522]: cluster 2024-04-13T07:26:35.890144+0000 mgr.y (mgr.14152) 240 : cluster 0 pgmap v214: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:39.733 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:39 smithi203 bash[23480]: cluster 2024-04-13T07:26:37.890799+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:39.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:39 smithi089 bash[21353]: cluster 2024-04-13T07:26:37.890799+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:39.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:39 smithi089 bash[31522]: cluster 2024-04-13T07:26:37.890799+0000 mgr.y (mgr.14152) 241 : cluster 0 pgmap v215: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:40.690 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:40 smithi203 bash[23480]: audit 2024-04-13T07:26:39.444144+0000 mon.a (mon.0) 491 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-13T07:26:40.690 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:40 smithi203 bash[23480]: audit 2024-04-13T07:26:39.445779+0000 mon.a (mon.0) 492 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:40.690 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:40 smithi203 bash[23480]: cephadm 2024-04-13T07:26:39.447222+0000 mgr.y (mgr.14152) 242 : cephadm 1 Deploying daemon osd.5 on smithi203 2024-04-13T07:26:40.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:40 smithi089 bash[31522]: audit 2024-04-13T07:26:39.444144+0000 mon.a (mon.0) 491 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-13T07:26:40.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:40 smithi089 bash[31522]: audit 2024-04-13T07:26:39.445779+0000 mon.a (mon.0) 492 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:40.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:40 smithi089 bash[31522]: cephadm 2024-04-13T07:26:39.447222+0000 mgr.y (mgr.14152) 242 : cephadm 1 Deploying daemon osd.5 on smithi203 2024-04-13T07:26:40.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:40 smithi089 bash[21353]: audit 2024-04-13T07:26:39.444144+0000 mon.a (mon.0) 491 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-13T07:26:40.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:40 smithi089 bash[21353]: audit 2024-04-13T07:26:39.445779+0000 mon.a (mon.0) 492 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:40.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:40 smithi089 bash[21353]: cephadm 2024-04-13T07:26:39.447222+0000 mgr.y (mgr.14152) 242 : cephadm 1 Deploying daemon osd.5 on smithi203 2024-04-13T07:26:41.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:41 smithi203 bash[23480]: cluster 2024-04-13T07:26:39.891425+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:41.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:41 smithi089 bash[21353]: cluster 2024-04-13T07:26:39.891425+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:41.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:41 smithi089 bash[31522]: cluster 2024-04-13T07:26:39.891425+0000 mgr.y (mgr.14152) 243 : cluster 0 pgmap v216: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:43.686 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:43 smithi203 bash[23480]: cluster 2024-04-13T07:26:41.892005+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:43.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:43 smithi089 bash[21353]: cluster 2024-04-13T07:26:41.892005+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:43.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:43 smithi089 bash[31522]: cluster 2024-04-13T07:26:41.892005+0000 mgr.y (mgr.14152) 244 : cluster 0 pgmap v217: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:43.936 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:26:43.937 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:26:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:26:43.937 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:26:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:26:44.210 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:44 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:26:44.210 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:26:44 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:26:44.210 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:26:44 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:26:44.518 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:44 smithi203 bash[23480]: audit 2024-04-13T07:26:44.204838+0000 mon.a (mon.0) 493 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:44.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:44 smithi203 bash[23480]: audit 2024-04-13T07:26:44.212567+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:44.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:44 smithi203 bash[23480]: audit 2024-04-13T07:26:44.223512+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:44.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:44 smithi089 bash[21353]: audit 2024-04-13T07:26:44.204838+0000 mon.a (mon.0) 493 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:44.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:44 smithi089 bash[21353]: audit 2024-04-13T07:26:44.212567+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:44.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:44 smithi089 bash[21353]: audit 2024-04-13T07:26:44.223512+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:44.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:44 smithi089 bash[31522]: audit 2024-04-13T07:26:44.204838+0000 mon.a (mon.0) 493 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:44.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:44 smithi089 bash[31522]: audit 2024-04-13T07:26:44.212567+0000 mon.a (mon.0) 494 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:44.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:44 smithi089 bash[31522]: audit 2024-04-13T07:26:44.223512+0000 mon.a (mon.0) 495 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:45.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:45 smithi203 bash[23480]: cluster 2024-04-13T07:26:43.892548+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:45 smithi089 bash[21353]: cluster 2024-04-13T07:26:43.892548+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:45.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:45 smithi089 bash[31522]: cluster 2024-04-13T07:26:43.892548+0000 mgr.y (mgr.14152) 245 : cluster 0 pgmap v218: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:47.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:47 smithi203 bash[23480]: cluster 2024-04-13T07:26:45.893102+0000 mgr.y (mgr.14152) 246 : cluster 0 pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:47.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:47 smithi089 bash[21353]: cluster 2024-04-13T07:26:45.893102+0000 mgr.y (mgr.14152) 246 : cluster 0 pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:47.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:47 smithi089 bash[31522]: cluster 2024-04-13T07:26:45.893102+0000 mgr.y (mgr.14152) 246 : cluster 0 pgmap v219: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:49.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:49 smithi203 bash[23480]: cluster 2024-04-13T07:26:47.893595+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:49.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:49 smithi089 bash[31522]: cluster 2024-04-13T07:26:47.893595+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:49.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:49 smithi089 bash[21353]: cluster 2024-04-13T07:26:47.893595+0000 mgr.y (mgr.14152) 247 : cluster 0 pgmap v220: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:50.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:50 smithi203 bash[23480]: audit 2024-04-13T07:26:49.397054+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:50.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:50 smithi203 bash[23480]: audit 2024-04-13T07:26:49.406811+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:50.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:50 smithi203 bash[23480]: audit 2024-04-13T07:26:49.934910+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:50.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:50 smithi203 bash[23480]: audit 2024-04-13T07:26:49.936384+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:26:50.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:50 smithi203 bash[23480]: audit 2024-04-13T07:26:49.946351+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:50.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[21353]: audit 2024-04-13T07:26:49.397054+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:50.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[21353]: audit 2024-04-13T07:26:49.406811+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:50.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[21353]: audit 2024-04-13T07:26:49.934910+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:50.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[21353]: audit 2024-04-13T07:26:49.936384+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:26:50.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[21353]: audit 2024-04-13T07:26:49.946351+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:50.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[31522]: audit 2024-04-13T07:26:49.397054+0000 mon.a (mon.0) 496 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:50.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[31522]: audit 2024-04-13T07:26:49.406811+0000 mon.a (mon.0) 497 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:50.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[31522]: audit 2024-04-13T07:26:49.934910+0000 mon.a (mon.0) 498 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:26:50.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[31522]: audit 2024-04-13T07:26:49.936384+0000 mon.a (mon.0) 499 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:26:50.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:50 smithi089 bash[31522]: audit 2024-04-13T07:26:49.946351+0000 mon.a (mon.0) 500 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:51.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:51 smithi203 bash[23480]: cluster 2024-04-13T07:26:49.894157+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:51.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:51 smithi203 bash[23480]: audit 2024-04-13T07:26:50.975023+0000 mon.b (mon.1) 15 : audit 1 from='osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-13T07:26:51.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:51 smithi203 bash[23480]: audit 2024-04-13T07:26:50.976055+0000 mon.a (mon.0) 501 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-13T07:26:51.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:51 smithi089 bash[21353]: cluster 2024-04-13T07:26:49.894157+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:51.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:51 smithi089 bash[21353]: audit 2024-04-13T07:26:50.975023+0000 mon.b (mon.1) 15 : audit 1 from='osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-13T07:26:51.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:51 smithi089 bash[21353]: audit 2024-04-13T07:26:50.976055+0000 mon.a (mon.0) 501 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-13T07:26:51.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:51 smithi089 bash[31522]: cluster 2024-04-13T07:26:49.894157+0000 mgr.y (mgr.14152) 248 : cluster 0 pgmap v221: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:51.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:51 smithi089 bash[31522]: audit 2024-04-13T07:26:50.975023+0000 mon.b (mon.1) 15 : audit 1 from='osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-13T07:26:51.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:51 smithi089 bash[31522]: audit 2024-04-13T07:26:50.976055+0000 mon.a (mon.0) 501 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2024-04-13T07:26:52.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:52 smithi203 bash[23480]: audit 2024-04-13T07:26:51.414135+0000 mon.a (mon.0) 502 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-13T07:26:52.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:52 smithi203 bash[23480]: cluster 2024-04-13T07:26:51.414376+0000 mon.a (mon.0) 503 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-04-13T07:26:52.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:52 smithi203 bash[23480]: audit 2024-04-13T07:26:51.415098+0000 mon.a (mon.0) 504 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:52.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:52 smithi203 bash[23480]: audit 2024-04-13T07:26:51.415228+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:52.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:52 smithi203 bash[23480]: audit 2024-04-13T07:26:51.428496+0000 mon.a (mon.0) 505 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:52.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[31522]: audit 2024-04-13T07:26:51.414135+0000 mon.a (mon.0) 502 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-13T07:26:52.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[31522]: cluster 2024-04-13T07:26:51.414376+0000 mon.a (mon.0) 503 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-04-13T07:26:52.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[31522]: audit 2024-04-13T07:26:51.415098+0000 mon.a (mon.0) 504 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:52.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[31522]: audit 2024-04-13T07:26:51.415228+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:52.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[31522]: audit 2024-04-13T07:26:51.428496+0000 mon.a (mon.0) 505 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:52.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[21353]: audit 2024-04-13T07:26:51.414135+0000 mon.a (mon.0) 502 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2024-04-13T07:26:52.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[21353]: cluster 2024-04-13T07:26:51.414376+0000 mon.a (mon.0) 503 : cluster 0 osdmap e37: 6 total, 5 up, 6 in 2024-04-13T07:26:52.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[21353]: audit 2024-04-13T07:26:51.415098+0000 mon.a (mon.0) 504 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:52.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[21353]: audit 2024-04-13T07:26:51.415228+0000 mon.b (mon.1) 16 : audit 1 from='osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:52.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:52 smithi089 bash[21353]: audit 2024-04-13T07:26:51.428496+0000 mon.a (mon.0) 505 : audit 1 from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:26:53.239 INFO:teuthology.orchestra.run.smithi203.stdout:Created osd(s) 5 on host 'smithi203' 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: cluster 2024-04-13T07:26:51.894852+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: audit 2024-04-13T07:26:52.439058+0000 mon.a (mon.0) 506 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: cluster 2024-04-13T07:26:52.439389+0000 mon.a (mon.0) 507 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: audit 2024-04-13T07:26:52.439901+0000 mon.a (mon.0) 508 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: audit 2024-04-13T07:26:52.444441+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: cluster 2024-04-13T07:26:52.798626+0000 mon.a (mon.0) 510 : cluster 1 osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809] boot 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: cluster 2024-04-13T07:26:52.798790+0000 mon.a (mon.0) 511 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: audit 2024-04-13T07:26:52.799112+0000 mon.a (mon.0) 512 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: audit 2024-04-13T07:26:53.190896+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: audit 2024-04-13T07:26:53.191395+0000 mon.a (mon.0) 514 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:53.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:53 smithi203 bash[23480]: audit 2024-04-13T07:26:53.209223+0000 mon.a (mon.0) 515 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: cluster 2024-04-13T07:26:51.894852+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: audit 2024-04-13T07:26:52.439058+0000 mon.a (mon.0) 506 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: cluster 2024-04-13T07:26:52.439389+0000 mon.a (mon.0) 507 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: audit 2024-04-13T07:26:52.439901+0000 mon.a (mon.0) 508 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: audit 2024-04-13T07:26:52.444441+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: cluster 2024-04-13T07:26:52.798626+0000 mon.a (mon.0) 510 : cluster 1 osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809] boot 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: cluster 2024-04-13T07:26:52.798790+0000 mon.a (mon.0) 511 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: audit 2024-04-13T07:26:52.799112+0000 mon.a (mon.0) 512 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: audit 2024-04-13T07:26:53.190896+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:53.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: audit 2024-04-13T07:26:53.191395+0000 mon.a (mon.0) 514 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[21353]: audit 2024-04-13T07:26:53.209223+0000 mon.a (mon.0) 515 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: cluster 2024-04-13T07:26:51.894852+0000 mgr.y (mgr.14152) 249 : cluster 0 pgmap v223: 1 pgs: 1 active+clean; 577 KiB data, 143 MiB used, 447 GiB / 447 GiB avail 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: audit 2024-04-13T07:26:52.439058+0000 mon.a (mon.0) 506 : audit 1 from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: cluster 2024-04-13T07:26:52.439389+0000 mon.a (mon.0) 507 : cluster 0 osdmap e38: 6 total, 5 up, 6 in 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: audit 2024-04-13T07:26:52.439901+0000 mon.a (mon.0) 508 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: audit 2024-04-13T07:26:52.444441+0000 mon.a (mon.0) 509 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: cluster 2024-04-13T07:26:52.798626+0000 mon.a (mon.0) 510 : cluster 1 osd.5 [v2:172.21.15.203:6808/209623809,v1:172.21.15.203:6809/209623809] boot 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: cluster 2024-04-13T07:26:52.798790+0000 mon.a (mon.0) 511 : cluster 0 osdmap e39: 6 total, 6 up, 6 in 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: audit 2024-04-13T07:26:52.799112+0000 mon.a (mon.0) 512 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: audit 2024-04-13T07:26:53.190896+0000 mon.a (mon.0) 513 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: audit 2024-04-13T07:26:53.191395+0000 mon.a (mon.0) 514 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:26:53.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:53 smithi089 bash[31522]: audit 2024-04-13T07:26:53.209223+0000 mon.a (mon.0) 515 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:26:53.981 DEBUG:teuthology.orchestra.run.smithi203:osd.5> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.5.service 2024-04-13T07:26:53.984 INFO:tasks.cephadm:Deploying osd.6 on smithi203 with /dev/vg_nvme/lv_2... 2024-04-13T07:26:53.985 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2024-04-13T07:26:55.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:54 smithi203 bash[23480]: cluster 2024-04-13T07:26:51.940620+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:26:55.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:54 smithi203 bash[23480]: cluster 2024-04-13T07:26:51.940703+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:26:55.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:54 smithi203 bash[23480]: cluster 2024-04-13T07:26:53.801020+0000 mon.a (mon.0) 516 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-04-13T07:26:55.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:54 smithi089 bash[21353]: cluster 2024-04-13T07:26:51.940620+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:26:55.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:54 smithi089 bash[21353]: cluster 2024-04-13T07:26:51.940703+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:26:55.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:54 smithi089 bash[21353]: cluster 2024-04-13T07:26:53.801020+0000 mon.a (mon.0) 516 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-04-13T07:26:55.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:54 smithi089 bash[31522]: cluster 2024-04-13T07:26:51.940620+0000 osd.5 (osd.5) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:26:55.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:54 smithi089 bash[31522]: cluster 2024-04-13T07:26:51.940703+0000 osd.5 (osd.5) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:26:55.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:54 smithi089 bash[31522]: cluster 2024-04-13T07:26:53.801020+0000 mon.a (mon.0) 516 : cluster 0 osdmap e40: 6 total, 6 up, 6 in 2024-04-13T07:26:56.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:55 smithi203 bash[23480]: cluster 2024-04-13T07:26:53.895343+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:26:56.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:55 smithi203 bash[23480]: cluster 2024-04-13T07:26:54.813978+0000 mon.a (mon.0) 517 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-04-13T07:26:56.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:55 smithi089 bash[21353]: cluster 2024-04-13T07:26:53.895343+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:26:56.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:55 smithi089 bash[21353]: cluster 2024-04-13T07:26:54.813978+0000 mon.a (mon.0) 517 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-04-13T07:26:56.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:55 smithi089 bash[31522]: cluster 2024-04-13T07:26:53.895343+0000 mgr.y (mgr.14152) 250 : cluster 0 pgmap v227: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:26:56.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:55 smithi089 bash[31522]: cluster 2024-04-13T07:26:54.813978+0000 mon.a (mon.0) 517 : cluster 0 osdmap e41: 6 total, 6 up, 6 in 2024-04-13T07:26:57.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:56 smithi203 bash[23480]: cluster 2024-04-13T07:26:55.819528+0000 mon.a (mon.0) 518 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-04-13T07:26:57.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:56 smithi089 bash[21353]: cluster 2024-04-13T07:26:55.819528+0000 mon.a (mon.0) 518 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-04-13T07:26:57.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:56 smithi089 bash[31522]: cluster 2024-04-13T07:26:55.819528+0000 mon.a (mon.0) 518 : cluster 0 osdmap e42: 6 total, 6 up, 6 in 2024-04-13T07:26:58.073 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:26:58.095 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:57 smithi203 bash[23480]: cluster 2024-04-13T07:26:55.895956+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v230: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:26:58.096 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:57 smithi203 bash[23480]: cluster 2024-04-13T07:26:56.814243+0000 mon.a (mon.0) 519 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-13T07:26:58.096 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:57 smithi203 bash[23480]: cluster 2024-04-13T07:26:56.833469+0000 mon.a (mon.0) 520 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-04-13T07:26:58.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:57 smithi089 bash[21353]: cluster 2024-04-13T07:26:55.895956+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v230: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:26:58.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:57 smithi089 bash[21353]: cluster 2024-04-13T07:26:56.814243+0000 mon.a (mon.0) 519 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-13T07:26:58.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:57 smithi089 bash[21353]: cluster 2024-04-13T07:26:56.833469+0000 mon.a (mon.0) 520 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-04-13T07:26:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:57 smithi089 bash[31522]: cluster 2024-04-13T07:26:55.895956+0000 mgr.y (mgr.14152) 251 : cluster 0 pgmap v230: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:26:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:57 smithi089 bash[31522]: cluster 2024-04-13T07:26:56.814243+0000 mon.a (mon.0) 519 : cluster 3 Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2024-04-13T07:26:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:57 smithi089 bash[31522]: cluster 2024-04-13T07:26:56.833469+0000 mon.a (mon.0) 520 : cluster 0 osdmap e43: 6 total, 6 up, 6 in 2024-04-13T07:27:00.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:26:59 smithi203 bash[23480]: cluster 2024-04-13T07:26:57.896625+0000 mgr.y (mgr.14152) 252 : cluster 0 pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:00.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:26:59 smithi089 bash[21353]: cluster 2024-04-13T07:26:57.896625+0000 mgr.y (mgr.14152) 252 : cluster 0 pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:00.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:26:59 smithi089 bash[31522]: cluster 2024-04-13T07:26:57.896625+0000 mgr.y (mgr.14152) 252 : cluster 0 pgmap v232: 1 pgs: 1 remapped+peering; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:01.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:00 smithi203 bash[23480]: cluster 2024-04-13T07:26:59.897342+0000 mgr.y (mgr.14152) 253 : cluster 0 pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:01.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:00 smithi203 bash[23480]: cluster 2024-04-13T07:27:00.832782+0000 mon.a (mon.0) 521 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-13T07:27:01.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:00 smithi203 bash[23480]: cluster 2024-04-13T07:27:00.832842+0000 mon.a (mon.0) 522 : cluster 1 Cluster is now healthy 2024-04-13T07:27:01.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:00 smithi089 bash[21353]: cluster 2024-04-13T07:26:59.897342+0000 mgr.y (mgr.14152) 253 : cluster 0 pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:01.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:00 smithi089 bash[21353]: cluster 2024-04-13T07:27:00.832782+0000 mon.a (mon.0) 521 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-13T07:27:01.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:00 smithi089 bash[21353]: cluster 2024-04-13T07:27:00.832842+0000 mon.a (mon.0) 522 : cluster 1 Cluster is now healthy 2024-04-13T07:27:01.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:00 smithi089 bash[31522]: cluster 2024-04-13T07:26:59.897342+0000 mgr.y (mgr.14152) 253 : cluster 0 pgmap v233: 1 pgs: 1 active+clean; 577 KiB data, 171 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:01.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:00 smithi089 bash[31522]: cluster 2024-04-13T07:27:00.832782+0000 mon.a (mon.0) 521 : cluster 1 Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2024-04-13T07:27:01.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:00 smithi089 bash[31522]: cluster 2024-04-13T07:27:00.832842+0000 mon.a (mon.0) 522 : cluster 1 Cluster is now healthy 2024-04-13T07:27:02.884 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping: /dev/vg_nvme/lv_2 2024-04-13T07:27:02.884 INFO:teuthology.orchestra.run.smithi203.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2024-04-13T07:27:02.884 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10+0 records in 2024-04-13T07:27:02.884 INFO:teuthology.orchestra.run.smithi203.stderr:10+0 records out 2024-04-13T07:27:02.884 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0147102 s, 713 MB/s 2024-04-13T07:27:02.884 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping successful for: 2024-04-13T07:27:03.192 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:02 smithi203 bash[23480]: cluster 2024-04-13T07:27:01.898062+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-13T07:27:03.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:02 smithi089 bash[21353]: cluster 2024-04-13T07:27:01.898062+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-13T07:27:03.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:02 smithi089 bash[31522]: cluster 2024-04-13T07:27:01.898062+0000 mgr.y (mgr.14152) 254 : cluster 0 pgmap v234: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 81 KiB/s, 0 objects/s recovering 2024-04-13T07:27:03.771 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi203:vg_nvme/lv_2 2024-04-13T07:27:05.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: cluster 2024-04-13T07:27:03.898659+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-13T07:27:05.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: cephadm 2024-04-13T07:27:04.203963+0000 mgr.y (mgr.14152) 256 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:27:05.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: audit 2024-04-13T07:27:04.215267+0000 mon.a (mon.0) 523 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: audit 2024-04-13T07:27:04.227207+0000 mon.a (mon.0) 524 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: audit 2024-04-13T07:27:04.229467+0000 mon.a (mon.0) 525 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:05.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: audit 2024-04-13T07:27:04.231237+0000 mon.a (mon.0) 526 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:05.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: cephadm 2024-04-13T07:27:04.232618+0000 mgr.y (mgr.14152) 257 : cephadm 1 Adjusting osd_memory_target on smithi203 to 8632M 2024-04-13T07:27:05.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: audit 2024-04-13T07:27:04.242733+0000 mon.a (mon.0) 527 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: audit 2024-04-13T07:27:04.244936+0000 mon.a (mon.0) 528 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:05.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: audit 2024-04-13T07:27:04.246711+0000 mon.a (mon.0) 529 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:05.520 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:05 smithi203 bash[23480]: audit 2024-04-13T07:27:04.258423+0000 mon.a (mon.0) 530 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: cluster 2024-04-13T07:27:03.898659+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-13T07:27:05.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: cephadm 2024-04-13T07:27:04.203963+0000 mgr.y (mgr.14152) 256 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: audit 2024-04-13T07:27:04.215267+0000 mon.a (mon.0) 523 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: audit 2024-04-13T07:27:04.227207+0000 mon.a (mon.0) 524 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: audit 2024-04-13T07:27:04.229467+0000 mon.a (mon.0) 525 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: audit 2024-04-13T07:27:04.231237+0000 mon.a (mon.0) 526 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: cephadm 2024-04-13T07:27:04.232618+0000 mgr.y (mgr.14152) 257 : cephadm 1 Adjusting osd_memory_target on smithi203 to 8632M 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: audit 2024-04-13T07:27:04.242733+0000 mon.a (mon.0) 527 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: audit 2024-04-13T07:27:04.244936+0000 mon.a (mon.0) 528 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: audit 2024-04-13T07:27:04.246711+0000 mon.a (mon.0) 529 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[21353]: audit 2024-04-13T07:27:04.258423+0000 mon.a (mon.0) 530 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: cluster 2024-04-13T07:27:03.898659+0000 mgr.y (mgr.14152) 255 : cluster 0 pgmap v235: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 71 KiB/s, 0 objects/s recovering 2024-04-13T07:27:05.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: cephadm 2024-04-13T07:27:04.203963+0000 mgr.y (mgr.14152) 256 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: audit 2024-04-13T07:27:04.215267+0000 mon.a (mon.0) 523 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: audit 2024-04-13T07:27:04.227207+0000 mon.a (mon.0) 524 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: audit 2024-04-13T07:27:04.229467+0000 mon.a (mon.0) 525 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: audit 2024-04-13T07:27:04.231237+0000 mon.a (mon.0) 526 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: cephadm 2024-04-13T07:27:04.232618+0000 mgr.y (mgr.14152) 257 : cephadm 1 Adjusting osd_memory_target on smithi203 to 8632M 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: audit 2024-04-13T07:27:04.242733+0000 mon.a (mon.0) 527 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: audit 2024-04-13T07:27:04.244936+0000 mon.a (mon.0) 528 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: audit 2024-04-13T07:27:04.246711+0000 mon.a (mon.0) 529 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:05.553 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:05 smithi089 bash[31522]: audit 2024-04-13T07:27:04.258423+0000 mon.a (mon.0) 530 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:07.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:07 smithi203 bash[23480]: cluster 2024-04-13T07:27:05.899239+0000 mgr.y (mgr.14152) 258 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-13T07:27:07.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:07 smithi089 bash[21353]: cluster 2024-04-13T07:27:05.899239+0000 mgr.y (mgr.14152) 258 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-13T07:27:07.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:07 smithi089 bash[31522]: cluster 2024-04-13T07:27:05.899239+0000 mgr.y (mgr.14152) 258 : cluster 0 pgmap v236: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2024-04-13T07:27:08.453 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:27:09.472 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:09 smithi203 bash[23480]: cluster 2024-04-13T07:27:07.899804+0000 mgr.y (mgr.14152) 259 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-13T07:27:09.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:09 smithi089 bash[21353]: cluster 2024-04-13T07:27:07.899804+0000 mgr.y (mgr.14152) 259 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-13T07:27:09.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:09 smithi089 bash[31522]: cluster 2024-04-13T07:27:07.899804+0000 mgr.y (mgr.14152) 259 : cluster 0 pgmap v237: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 52 KiB/s, 0 objects/s recovering 2024-04-13T07:27:11.518 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:11 smithi203 bash[23480]: cluster 2024-04-13T07:27:09.900382+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-13T07:27:11.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:11 smithi089 bash[21353]: cluster 2024-04-13T07:27:09.900382+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-13T07:27:11.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:11 smithi089 bash[31522]: cluster 2024-04-13T07:27:09.900382+0000 mgr.y (mgr.14152) 260 : cluster 0 pgmap v238: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-13T07:27:12.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:12 smithi203 bash[23480]: audit 2024-04-13T07:27:11.311779+0000 mgr.y (mgr.14152) 261 : audit 0 from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:27:12.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:12 smithi203 bash[23480]: audit 2024-04-13T07:27:11.318670+0000 mon.a (mon.0) 531 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:27:12.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:12 smithi203 bash[23480]: audit 2024-04-13T07:27:11.327107+0000 mon.a (mon.0) 532 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:27:12.519 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:12 smithi203 bash[23480]: audit 2024-04-13T07:27:11.328796+0000 mon.a (mon.0) 533 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:12.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:12 smithi089 bash[21353]: audit 2024-04-13T07:27:11.311779+0000 mgr.y (mgr.14152) 261 : audit 0 from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:27:12.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:12 smithi089 bash[21353]: audit 2024-04-13T07:27:11.318670+0000 mon.a (mon.0) 531 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:27:12.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:12 smithi089 bash[21353]: audit 2024-04-13T07:27:11.327107+0000 mon.a (mon.0) 532 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:27:12.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:12 smithi089 bash[21353]: audit 2024-04-13T07:27:11.328796+0000 mon.a (mon.0) 533 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:12.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:12 smithi089 bash[31522]: audit 2024-04-13T07:27:11.311779+0000 mgr.y (mgr.14152) 261 : audit 0 from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:27:12.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:12 smithi089 bash[31522]: audit 2024-04-13T07:27:11.318670+0000 mon.a (mon.0) 531 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:27:12.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:12 smithi089 bash[31522]: audit 2024-04-13T07:27:11.327107+0000 mon.a (mon.0) 532 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:27:12.552 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:12 smithi089 bash[31522]: audit 2024-04-13T07:27:11.328796+0000 mon.a (mon.0) 533 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:13.518 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:13 smithi203 bash[23480]: cluster 2024-04-13T07:27:11.900927+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:13.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:13 smithi089 bash[21353]: cluster 2024-04-13T07:27:11.900927+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:13.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:13 smithi089 bash[31522]: cluster 2024-04-13T07:27:11.900927+0000 mgr.y (mgr.14152) 262 : cluster 0 pgmap v239: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:15.518 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:15 smithi203 bash[23480]: cluster 2024-04-13T07:27:13.901449+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:15.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:15 smithi089 bash[21353]: cluster 2024-04-13T07:27:13.901449+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:15.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:15 smithi089 bash[31522]: cluster 2024-04-13T07:27:13.901449+0000 mgr.y (mgr.14152) 263 : cluster 0 pgmap v240: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:17.463 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:17 smithi203 bash[23480]: cluster 2024-04-13T07:27:15.902007+0000 mgr.y (mgr.14152) 264 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:17.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:17 smithi089 bash[31522]: cluster 2024-04-13T07:27:15.902007+0000 mgr.y (mgr.14152) 264 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:17.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:17 smithi089 bash[21353]: cluster 2024-04-13T07:27:15.902007+0000 mgr.y (mgr.14152) 264 : cluster 0 pgmap v241: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:20.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:19 smithi203 bash[23480]: cluster 2024-04-13T07:27:17.902570+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:20.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:19 smithi203 bash[23480]: audit 2024-04-13T07:27:19.687755+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.203:0/2212376000' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]: dispatch 2024-04-13T07:27:20.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:19 smithi203 bash[23480]: audit 2024-04-13T07:27:19.688834+0000 mon.a (mon.0) 534 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]: dispatch 2024-04-13T07:27:20.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:19 smithi203 bash[23480]: audit 2024-04-13T07:27:19.699662+0000 mon.a (mon.0) 535 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]': finished 2024-04-13T07:27:20.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:19 smithi203 bash[23480]: cluster 2024-04-13T07:27:19.699890+0000 mon.a (mon.0) 536 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-04-13T07:27:20.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:19 smithi203 bash[23480]: audit 2024-04-13T07:27:19.700135+0000 mon.a (mon.0) 537 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:20.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[21353]: cluster 2024-04-13T07:27:17.902570+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:20.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[21353]: audit 2024-04-13T07:27:19.687755+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.203:0/2212376000' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]: dispatch 2024-04-13T07:27:20.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[21353]: audit 2024-04-13T07:27:19.688834+0000 mon.a (mon.0) 534 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]: dispatch 2024-04-13T07:27:20.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[21353]: audit 2024-04-13T07:27:19.699662+0000 mon.a (mon.0) 535 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]': finished 2024-04-13T07:27:20.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[21353]: cluster 2024-04-13T07:27:19.699890+0000 mon.a (mon.0) 536 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-04-13T07:27:20.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[21353]: audit 2024-04-13T07:27:19.700135+0000 mon.a (mon.0) 537 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:20.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[31522]: cluster 2024-04-13T07:27:17.902570+0000 mgr.y (mgr.14152) 265 : cluster 0 pgmap v242: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:20.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[31522]: audit 2024-04-13T07:27:19.687755+0000 mon.b (mon.1) 17 : audit 1 from='client.? 172.21.15.203:0/2212376000' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]: dispatch 2024-04-13T07:27:20.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[31522]: audit 2024-04-13T07:27:19.688834+0000 mon.a (mon.0) 534 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]: dispatch 2024-04-13T07:27:20.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[31522]: audit 2024-04-13T07:27:19.699662+0000 mon.a (mon.0) 535 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "91bbf36b-799c-49df-8fbf-42a958699642"}]': finished 2024-04-13T07:27:20.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[31522]: cluster 2024-04-13T07:27:19.699890+0000 mon.a (mon.0) 536 : cluster 0 osdmap e44: 7 total, 6 up, 7 in 2024-04-13T07:27:20.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:19 smithi089 bash[31522]: audit 2024-04-13T07:27:19.700135+0000 mon.a (mon.0) 537 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:21.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:20 smithi203 bash[23480]: audit 2024-04-13T07:27:20.466987+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.203:0/3060712639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:27:21.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:20 smithi089 bash[21353]: audit 2024-04-13T07:27:20.466987+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.203:0/3060712639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:27:21.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:20 smithi089 bash[31522]: audit 2024-04-13T07:27:20.466987+0000 mon.b (mon.1) 18 : audit 0 from='client.? 172.21.15.203:0/3060712639' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:27:22.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:21 smithi203 bash[23480]: cluster 2024-04-13T07:27:19.903098+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:22.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:21 smithi089 bash[21353]: cluster 2024-04-13T07:27:19.903098+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:22.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:21 smithi089 bash[31522]: cluster 2024-04-13T07:27:19.903098+0000 mgr.y (mgr.14152) 266 : cluster 0 pgmap v244: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:24.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:23 smithi203 bash[23480]: cluster 2024-04-13T07:27:21.903626+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:24.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:23 smithi089 bash[21353]: cluster 2024-04-13T07:27:21.903626+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:24.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:23 smithi089 bash[31522]: cluster 2024-04-13T07:27:21.903626+0000 mgr.y (mgr.14152) 267 : cluster 0 pgmap v245: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:26.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:25 smithi203 bash[23480]: cluster 2024-04-13T07:27:23.904193+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:26.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:25 smithi089 bash[21353]: cluster 2024-04-13T07:27:23.904193+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:26.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:25 smithi089 bash[31522]: cluster 2024-04-13T07:27:23.904193+0000 mgr.y (mgr.14152) 268 : cluster 0 pgmap v246: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:28.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:27 smithi203 bash[23480]: cluster 2024-04-13T07:27:25.904807+0000 mgr.y (mgr.14152) 269 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:28.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:27 smithi089 bash[21353]: cluster 2024-04-13T07:27:25.904807+0000 mgr.y (mgr.14152) 269 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:28.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:27 smithi089 bash[31522]: cluster 2024-04-13T07:27:25.904807+0000 mgr.y (mgr.14152) 269 : cluster 0 pgmap v247: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:30.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:29 smithi203 bash[23480]: cluster 2024-04-13T07:27:27.905359+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:30.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:29 smithi089 bash[31522]: cluster 2024-04-13T07:27:27.905359+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:30.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:29 smithi089 bash[21353]: cluster 2024-04-13T07:27:27.905359+0000 mgr.y (mgr.14152) 270 : cluster 0 pgmap v248: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:32.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:31 smithi203 bash[23480]: cluster 2024-04-13T07:27:29.905962+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:32.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:31 smithi089 bash[21353]: cluster 2024-04-13T07:27:29.905962+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:32.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:31 smithi089 bash[31522]: cluster 2024-04-13T07:27:29.905962+0000 mgr.y (mgr.14152) 271 : cluster 0 pgmap v249: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:33.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:32 smithi203 bash[23480]: audit 2024-04-13T07:27:32.319557+0000 mon.a (mon.0) 538 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-13T07:27:33.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:32 smithi203 bash[23480]: audit 2024-04-13T07:27:32.321278+0000 mon.a (mon.0) 539 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:33.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:32 smithi089 bash[21353]: audit 2024-04-13T07:27:32.319557+0000 mon.a (mon.0) 538 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-13T07:27:33.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:32 smithi089 bash[21353]: audit 2024-04-13T07:27:32.321278+0000 mon.a (mon.0) 539 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:33.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:32 smithi089 bash[31522]: audit 2024-04-13T07:27:32.319557+0000 mon.a (mon.0) 538 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-13T07:27:33.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:32 smithi089 bash[31522]: audit 2024-04-13T07:27:32.321278+0000 mon.a (mon.0) 539 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:34.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:33 smithi203 bash[23480]: cluster 2024-04-13T07:27:31.906493+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:34.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:33 smithi203 bash[23480]: cephadm 2024-04-13T07:27:32.322832+0000 mgr.y (mgr.14152) 273 : cephadm 1 Deploying daemon osd.6 on smithi203 2024-04-13T07:27:34.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:33 smithi089 bash[21353]: cluster 2024-04-13T07:27:31.906493+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:34.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:33 smithi089 bash[21353]: cephadm 2024-04-13T07:27:32.322832+0000 mgr.y (mgr.14152) 273 : cephadm 1 Deploying daemon osd.6 on smithi203 2024-04-13T07:27:34.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:33 smithi089 bash[31522]: cluster 2024-04-13T07:27:31.906493+0000 mgr.y (mgr.14152) 272 : cluster 0 pgmap v250: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:34.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:33 smithi089 bash[31522]: cephadm 2024-04-13T07:27:32.322832+0000 mgr.y (mgr.14152) 273 : cephadm 1 Deploying daemon osd.6 on smithi203 2024-04-13T07:27:36.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:35 smithi203 bash[23480]: cluster 2024-04-13T07:27:33.907175+0000 mgr.y (mgr.14152) 274 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:36.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:35 smithi089 bash[21353]: cluster 2024-04-13T07:27:33.907175+0000 mgr.y (mgr.14152) 274 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:36.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:35 smithi089 bash[31522]: cluster 2024-04-13T07:27:33.907175+0000 mgr.y (mgr.14152) 274 : cluster 0 pgmap v251: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:36.975 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:27:36 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:27:36.975 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:36 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:27:36.976 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:27:36 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:27:36.976 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:27:36 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:27:37.257 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:37 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:27:37.257 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:27:37 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:27:37.257 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:27:37 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:27:37.257 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:27:37 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:27:38.210 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:37 smithi203 bash[23480]: cluster 2024-04-13T07:27:35.907710+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:38.210 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:37 smithi203 bash[23480]: audit 2024-04-13T07:27:37.263336+0000 mon.a (mon.0) 540 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:38.210 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:37 smithi203 bash[23480]: audit 2024-04-13T07:27:37.273093+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:38.210 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:37 smithi203 bash[23480]: audit 2024-04-13T07:27:37.284894+0000 mon.a (mon.0) 542 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:27:38.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:37 smithi089 bash[21353]: cluster 2024-04-13T07:27:35.907710+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:38.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:37 smithi089 bash[21353]: audit 2024-04-13T07:27:37.263336+0000 mon.a (mon.0) 540 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:38.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:37 smithi089 bash[21353]: audit 2024-04-13T07:27:37.273093+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:38.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:37 smithi089 bash[21353]: audit 2024-04-13T07:27:37.284894+0000 mon.a (mon.0) 542 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:27:38.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:37 smithi089 bash[31522]: cluster 2024-04-13T07:27:35.907710+0000 mgr.y (mgr.14152) 275 : cluster 0 pgmap v252: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:38.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:37 smithi089 bash[31522]: audit 2024-04-13T07:27:37.263336+0000 mon.a (mon.0) 540 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:38.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:37 smithi089 bash[31522]: audit 2024-04-13T07:27:37.273093+0000 mon.a (mon.0) 541 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:38.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:37 smithi089 bash[31522]: audit 2024-04-13T07:27:37.284894+0000 mon.a (mon.0) 542 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:27:40.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:39 smithi203 bash[23480]: cluster 2024-04-13T07:27:37.908196+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:40.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:39 smithi089 bash[31522]: cluster 2024-04-13T07:27:37.908196+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:40.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:39 smithi089 bash[21353]: cluster 2024-04-13T07:27:37.908196+0000 mgr.y (mgr.14152) 276 : cluster 0 pgmap v253: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:42.150 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:41 smithi203 bash[23480]: cluster 2024-04-13T07:27:39.908776+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:42.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:41 smithi089 bash[21353]: cluster 2024-04-13T07:27:39.908776+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:42.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:41 smithi089 bash[31522]: cluster 2024-04-13T07:27:39.908776+0000 mgr.y (mgr.14152) 277 : cluster 0 pgmap v254: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:43.788 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:43 smithi203 bash[23480]: cluster 2024-04-13T07:27:41.909335+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:43.788 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:43 smithi203 bash[23480]: audit 2024-04-13T07:27:42.480307+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:43.788 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:43 smithi203 bash[23480]: audit 2024-04-13T07:27:42.490228+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:43.788 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:43 smithi203 bash[23480]: audit 2024-04-13T07:27:42.492242+0000 mon.a (mon.0) 545 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:43.788 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:43 smithi203 bash[23480]: audit 2024-04-13T07:27:42.494099+0000 mon.a (mon.0) 546 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:43.788 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:43 smithi203 bash[23480]: audit 2024-04-13T07:27:42.504462+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:43.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[21353]: cluster 2024-04-13T07:27:41.909335+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:43.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[21353]: audit 2024-04-13T07:27:42.480307+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:43.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[21353]: audit 2024-04-13T07:27:42.490228+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:43.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[21353]: audit 2024-04-13T07:27:42.492242+0000 mon.a (mon.0) 545 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:43.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[21353]: audit 2024-04-13T07:27:42.494099+0000 mon.a (mon.0) 546 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:43.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[21353]: audit 2024-04-13T07:27:42.504462+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:43.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[31522]: cluster 2024-04-13T07:27:41.909335+0000 mgr.y (mgr.14152) 278 : cluster 0 pgmap v255: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:43.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[31522]: audit 2024-04-13T07:27:42.480307+0000 mon.a (mon.0) 543 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:43.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[31522]: audit 2024-04-13T07:27:42.490228+0000 mon.a (mon.0) 544 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:43.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[31522]: audit 2024-04-13T07:27:42.492242+0000 mon.a (mon.0) 545 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:43.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[31522]: audit 2024-04-13T07:27:42.494099+0000 mon.a (mon.0) 546 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:43.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:43 smithi089 bash[31522]: audit 2024-04-13T07:27:42.504462+0000 mon.a (mon.0) 547 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:44.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:44 smithi203 bash[23480]: audit 2024-04-13T07:27:43.962144+0000 mon.b (mon.1) 19 : audit 1 from='osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-13T07:27:44.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:44 smithi203 bash[23480]: audit 2024-04-13T07:27:43.963230+0000 mon.a (mon.0) 548 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-13T07:27:44.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:44 smithi089 bash[21353]: audit 2024-04-13T07:27:43.962144+0000 mon.b (mon.1) 19 : audit 1 from='osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-13T07:27:44.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:44 smithi089 bash[21353]: audit 2024-04-13T07:27:43.963230+0000 mon.a (mon.0) 548 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-13T07:27:44.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:44 smithi089 bash[31522]: audit 2024-04-13T07:27:43.962144+0000 mon.b (mon.1) 19 : audit 1 from='osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-13T07:27:44.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:44 smithi089 bash[31522]: audit 2024-04-13T07:27:43.963230+0000 mon.a (mon.0) 548 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2024-04-13T07:27:45.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:45 smithi203 bash[23480]: cluster 2024-04-13T07:27:43.909907+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:45.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:45 smithi203 bash[23480]: audit 2024-04-13T07:27:44.492867+0000 mon.a (mon.0) 549 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-13T07:27:45.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:45 smithi203 bash[23480]: cluster 2024-04-13T07:27:44.493080+0000 mon.a (mon.0) 550 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-04-13T07:27:45.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:45 smithi203 bash[23480]: audit 2024-04-13T07:27:44.493381+0000 mon.a (mon.0) 551 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:45.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:45 smithi203 bash[23480]: audit 2024-04-13T07:27:44.494033+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:27:45.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:45 smithi203 bash[23480]: audit 2024-04-13T07:27:44.495242+0000 mon.a (mon.0) 552 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:27:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[21353]: cluster 2024-04-13T07:27:43.909907+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[21353]: audit 2024-04-13T07:27:44.492867+0000 mon.a (mon.0) 549 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-13T07:27:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[21353]: cluster 2024-04-13T07:27:44.493080+0000 mon.a (mon.0) 550 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-04-13T07:27:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[21353]: audit 2024-04-13T07:27:44.493381+0000 mon.a (mon.0) 551 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[21353]: audit 2024-04-13T07:27:44.494033+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:27:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[21353]: audit 2024-04-13T07:27:44.495242+0000 mon.a (mon.0) 552 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:27:45.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[31522]: cluster 2024-04-13T07:27:43.909907+0000 mgr.y (mgr.14152) 279 : cluster 0 pgmap v256: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:45.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[31522]: audit 2024-04-13T07:27:44.492867+0000 mon.a (mon.0) 549 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2024-04-13T07:27:45.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[31522]: cluster 2024-04-13T07:27:44.493080+0000 mon.a (mon.0) 550 : cluster 0 osdmap e45: 7 total, 6 up, 7 in 2024-04-13T07:27:45.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[31522]: audit 2024-04-13T07:27:44.493381+0000 mon.a (mon.0) 551 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:45.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[31522]: audit 2024-04-13T07:27:44.494033+0000 mon.b (mon.1) 20 : audit 1 from='osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:27:45.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:45 smithi089 bash[31522]: audit 2024-04-13T07:27:44.495242+0000 mon.a (mon.0) 552 : audit 1 from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:27:46.333 INFO:teuthology.orchestra.run.smithi203.stdout:Created osd(s) 6 on host 'smithi203' 2024-04-13T07:27:46.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:46 smithi203 bash[23480]: audit 2024-04-13T07:27:45.496170+0000 mon.a (mon.0) 553 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:27:46.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:46 smithi203 bash[23480]: cluster 2024-04-13T07:27:45.496473+0000 mon.a (mon.0) 554 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-04-13T07:27:46.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:46 smithi203 bash[23480]: audit 2024-04-13T07:27:45.497288+0000 mon.a (mon.0) 555 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:46.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:46 smithi203 bash[23480]: audit 2024-04-13T07:27:45.505983+0000 mon.a (mon.0) 556 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:46.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:46 smithi203 bash[23480]: audit 2024-04-13T07:27:46.321713+0000 mon.a (mon.0) 557 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:46.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:46 smithi203 bash[23480]: audit 2024-04-13T07:27:46.331394+0000 mon.a (mon.0) 558 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:46.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:46 smithi203 bash[23480]: audit 2024-04-13T07:27:46.351072+0000 mon.a (mon.0) 559 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[21353]: audit 2024-04-13T07:27:45.496170+0000 mon.a (mon.0) 553 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[21353]: cluster 2024-04-13T07:27:45.496473+0000 mon.a (mon.0) 554 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[21353]: audit 2024-04-13T07:27:45.497288+0000 mon.a (mon.0) 555 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[21353]: audit 2024-04-13T07:27:45.505983+0000 mon.a (mon.0) 556 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[21353]: audit 2024-04-13T07:27:46.321713+0000 mon.a (mon.0) 557 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[21353]: audit 2024-04-13T07:27:46.331394+0000 mon.a (mon.0) 558 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[21353]: audit 2024-04-13T07:27:46.351072+0000 mon.a (mon.0) 559 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[31522]: audit 2024-04-13T07:27:45.496170+0000 mon.a (mon.0) 553 : audit 1 from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[31522]: cluster 2024-04-13T07:27:45.496473+0000 mon.a (mon.0) 554 : cluster 0 osdmap e46: 7 total, 6 up, 7 in 2024-04-13T07:27:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[31522]: audit 2024-04-13T07:27:45.497288+0000 mon.a (mon.0) 555 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:46.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[31522]: audit 2024-04-13T07:27:45.505983+0000 mon.a (mon.0) 556 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:46.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[31522]: audit 2024-04-13T07:27:46.321713+0000 mon.a (mon.0) 557 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:46.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[31522]: audit 2024-04-13T07:27:46.331394+0000 mon.a (mon.0) 558 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:46.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:46 smithi089 bash[31522]: audit 2024-04-13T07:27:46.351072+0000 mon.a (mon.0) 559 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:27:47.064 DEBUG:teuthology.orchestra.run.smithi203:osd.6> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.6.service 2024-04-13T07:27:47.067 INFO:tasks.cephadm:Deploying osd.7 on smithi203 with /dev/vg_nvme/lv_1... 2024-04-13T07:27:47.067 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2024-04-13T07:27:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:47 smithi203 bash[23480]: cluster 2024-04-13T07:27:44.965972+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:27:47.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:47 smithi203 bash[23480]: cluster 2024-04-13T07:27:44.966130+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:27:47.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:47 smithi203 bash[23480]: cluster 2024-04-13T07:27:45.910460+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:47.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:47 smithi203 bash[23480]: cluster 2024-04-13T07:27:46.499674+0000 mon.a (mon.0) 560 : cluster 1 osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907] boot 2024-04-13T07:27:47.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:47 smithi203 bash[23480]: cluster 2024-04-13T07:27:46.499808+0000 mon.a (mon.0) 561 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-04-13T07:27:47.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:47 smithi203 bash[23480]: audit 2024-04-13T07:27:46.500217+0000 mon.a (mon.0) 562 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:47.770 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:47 smithi203 bash[23480]: cluster 2024-04-13T07:27:47.500917+0000 mon.a (mon.0) 563 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-04-13T07:27:47.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[21353]: cluster 2024-04-13T07:27:44.965972+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:27:47.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[21353]: cluster 2024-04-13T07:27:44.966130+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:27:47.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[21353]: cluster 2024-04-13T07:27:45.910460+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:47.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[21353]: cluster 2024-04-13T07:27:46.499674+0000 mon.a (mon.0) 560 : cluster 1 osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907] boot 2024-04-13T07:27:47.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[21353]: cluster 2024-04-13T07:27:46.499808+0000 mon.a (mon.0) 561 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-04-13T07:27:47.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[21353]: audit 2024-04-13T07:27:46.500217+0000 mon.a (mon.0) 562 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:47.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[21353]: cluster 2024-04-13T07:27:47.500917+0000 mon.a (mon.0) 563 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-04-13T07:27:47.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[31522]: cluster 2024-04-13T07:27:44.965972+0000 osd.6 (osd.6) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:27:47.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[31522]: cluster 2024-04-13T07:27:44.966130+0000 osd.6 (osd.6) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:27:47.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[31522]: cluster 2024-04-13T07:27:45.910460+0000 mgr.y (mgr.14152) 280 : cluster 0 pgmap v259: 1 pgs: 1 active+clean; 577 KiB data, 170 MiB used, 536 GiB / 536 GiB avail 2024-04-13T07:27:47.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[31522]: cluster 2024-04-13T07:27:46.499674+0000 mon.a (mon.0) 560 : cluster 1 osd.6 [v2:172.21.15.203:6816/2138665907,v1:172.21.15.203:6817/2138665907] boot 2024-04-13T07:27:47.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[31522]: cluster 2024-04-13T07:27:46.499808+0000 mon.a (mon.0) 561 : cluster 0 osdmap e47: 7 total, 7 up, 7 in 2024-04-13T07:27:47.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[31522]: audit 2024-04-13T07:27:46.500217+0000 mon.a (mon.0) 562 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:27:47.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:47 smithi089 bash[31522]: cluster 2024-04-13T07:27:47.500917+0000 mon.a (mon.0) 563 : cluster 0 osdmap e48: 7 total, 7 up, 7 in 2024-04-13T07:27:49.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:49 smithi203 bash[23480]: cluster 2024-04-13T07:27:47.911008+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:27:49.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:49 smithi203 bash[23480]: cluster 2024-04-13T07:27:48.503809+0000 mon.a (mon.0) 564 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-04-13T07:27:49.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:49 smithi089 bash[21353]: cluster 2024-04-13T07:27:47.911008+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:27:49.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:49 smithi089 bash[21353]: cluster 2024-04-13T07:27:48.503809+0000 mon.a (mon.0) 564 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-04-13T07:27:49.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:49 smithi089 bash[31522]: cluster 2024-04-13T07:27:47.911008+0000 mgr.y (mgr.14152) 281 : cluster 0 pgmap v262: 1 pgs: 1 active+clean; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:27:49.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:49 smithi089 bash[31522]: cluster 2024-04-13T07:27:48.503809+0000 mon.a (mon.0) 564 : cluster 0 osdmap e49: 7 total, 7 up, 7 in 2024-04-13T07:27:50.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:50 smithi089 bash[21353]: cluster 2024-04-13T07:27:49.524218+0000 mon.a (mon.0) 565 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-04-13T07:27:50.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:50 smithi089 bash[31522]: cluster 2024-04-13T07:27:49.524218+0000 mon.a (mon.0) 565 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-04-13T07:27:51.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:50 smithi203 bash[23480]: cluster 2024-04-13T07:27:49.524218+0000 mon.a (mon.0) 565 : cluster 0 osdmap e50: 7 total, 7 up, 7 in 2024-04-13T07:27:51.186 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:27:51.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:51 smithi089 bash[21353]: cluster 2024-04-13T07:27:49.911582+0000 mgr.y (mgr.14152) 282 : cluster 0 pgmap v265: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-13T07:27:51.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:51 smithi089 bash[21353]: cluster 2024-04-13T07:27:50.537173+0000 mon.a (mon.0) 566 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-04-13T07:27:51.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:51 smithi089 bash[31522]: cluster 2024-04-13T07:27:49.911582+0000 mgr.y (mgr.14152) 282 : cluster 0 pgmap v265: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-13T07:27:51.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:51 smithi089 bash[31522]: cluster 2024-04-13T07:27:50.537173+0000 mon.a (mon.0) 566 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-04-13T07:27:52.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:51 smithi203 bash[23480]: cluster 2024-04-13T07:27:49.911582+0000 mgr.y (mgr.14152) 282 : cluster 0 pgmap v265: 1 pgs: 1 active+undersized+remapped; 577 KiB data, 197 MiB used, 626 GiB / 626 GiB avail; 144 KiB/s, 0 objects/s recovering 2024-04-13T07:27:52.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:51 smithi203 bash[23480]: cluster 2024-04-13T07:27:50.537173+0000 mon.a (mon.0) 566 : cluster 0 osdmap e51: 7 total, 7 up, 7 in 2024-04-13T07:27:53.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:53 smithi089 bash[31522]: cluster 2024-04-13T07:27:51.912196+0000 mgr.y (mgr.14152) 283 : cluster 0 pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2024-04-13T07:27:53.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:53 smithi089 bash[21353]: cluster 2024-04-13T07:27:51.912196+0000 mgr.y (mgr.14152) 283 : cluster 0 pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2024-04-13T07:27:54.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:53 smithi203 bash[23480]: cluster 2024-04-13T07:27:51.912196+0000 mgr.y (mgr.14152) 283 : cluster 0 pgmap v267: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 131 KiB/s, 0 objects/s recovering 2024-04-13T07:27:55.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:55 smithi203 bash[23480]: cluster 2024-04-13T07:27:53.912771+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:27:55.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:55 smithi089 bash[21353]: cluster 2024-04-13T07:27:53.912771+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:27:55.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:55 smithi089 bash[31522]: cluster 2024-04-13T07:27:53.912771+0000 mgr.y (mgr.14152) 284 : cluster 0 pgmap v268: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:27:56.065 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping: /dev/vg_nvme/lv_1 2024-04-13T07:27:56.065 INFO:teuthology.orchestra.run.smithi203.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2024-04-13T07:27:56.065 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10+0 records in 2024-04-13T07:27:56.065 INFO:teuthology.orchestra.run.smithi203.stderr:10+0 records out 2024-04-13T07:27:56.065 INFO:teuthology.orchestra.run.smithi203.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0154654 s, 678 MB/s 2024-04-13T07:27:56.065 INFO:teuthology.orchestra.run.smithi203.stderr:--> Zapping successful for: 2024-04-13T07:27:57.048 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch daemon add osd smithi203:vg_nvme/lv_1 2024-04-13T07:27:57.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: cluster 2024-04-13T07:27:55.913318+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-13T07:27:57.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: audit 2024-04-13T07:27:57.508961+0000 mon.a (mon.0) 567 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:57.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: audit 2024-04-13T07:27:57.519084+0000 mon.a (mon.0) 568 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:57.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: audit 2024-04-13T07:27:57.521786+0000 mon.a (mon.0) 569 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:57.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: audit 2024-04-13T07:27:57.523684+0000 mon.a (mon.0) 570 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:57.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: audit 2024-04-13T07:27:57.525528+0000 mon.a (mon.0) 571 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: audit 2024-04-13T07:27:57.535263+0000 mon.a (mon.0) 572 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: audit 2024-04-13T07:27:57.537385+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[21353]: audit 2024-04-13T07:27:57.539126+0000 mon.a (mon.0) 574 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: cluster 2024-04-13T07:27:55.913318+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: audit 2024-04-13T07:27:57.508961+0000 mon.a (mon.0) 567 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: audit 2024-04-13T07:27:57.519084+0000 mon.a (mon.0) 568 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: audit 2024-04-13T07:27:57.521786+0000 mon.a (mon.0) 569 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: audit 2024-04-13T07:27:57.523684+0000 mon.a (mon.0) 570 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: audit 2024-04-13T07:27:57.525528+0000 mon.a (mon.0) 571 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: audit 2024-04-13T07:27:57.535263+0000 mon.a (mon.0) 572 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:57.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: audit 2024-04-13T07:27:57.537385+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:57.803 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:57 smithi089 bash[31522]: audit 2024-04-13T07:27:57.539126+0000 mon.a (mon.0) 574 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:58.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: cluster 2024-04-13T07:27:55.913318+0000 mgr.y (mgr.14152) 285 : cluster 0 pgmap v269: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-13T07:27:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: audit 2024-04-13T07:27:57.508961+0000 mon.a (mon.0) 567 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: audit 2024-04-13T07:27:57.519084+0000 mon.a (mon.0) 568 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: audit 2024-04-13T07:27:57.521786+0000 mon.a (mon.0) 569 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: audit 2024-04-13T07:27:57.523684+0000 mon.a (mon.0) 570 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: audit 2024-04-13T07:27:57.525528+0000 mon.a (mon.0) 571 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:27:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: audit 2024-04-13T07:27:57.535263+0000 mon.a (mon.0) 572 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: audit 2024-04-13T07:27:57.537385+0000 mon.a (mon.0) 573 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:27:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:57 smithi203 bash[23480]: audit 2024-04-13T07:27:57.539126+0000 mon.a (mon.0) 574 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:27:59.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:58 smithi203 bash[23480]: cephadm 2024-04-13T07:27:57.499629+0000 mgr.y (mgr.14152) 286 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:27:59.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:58 smithi203 bash[23480]: cephadm 2024-04-13T07:27:57.526928+0000 mgr.y (mgr.14152) 287 : cephadm 1 Adjusting osd_memory_target on smithi203 to 5755M 2024-04-13T07:27:59.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:58 smithi203 bash[23480]: audit 2024-04-13T07:27:57.559260+0000 mon.a (mon.0) 575 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:59.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:58 smithi089 bash[21353]: cephadm 2024-04-13T07:27:57.499629+0000 mgr.y (mgr.14152) 286 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:27:59.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:58 smithi089 bash[21353]: cephadm 2024-04-13T07:27:57.526928+0000 mgr.y (mgr.14152) 287 : cephadm 1 Adjusting osd_memory_target on smithi203 to 5755M 2024-04-13T07:27:59.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:58 smithi089 bash[21353]: audit 2024-04-13T07:27:57.559260+0000 mon.a (mon.0) 575 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:27:59.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:58 smithi089 bash[31522]: cephadm 2024-04-13T07:27:57.499629+0000 mgr.y (mgr.14152) 286 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:27:59.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:58 smithi089 bash[31522]: cephadm 2024-04-13T07:27:57.526928+0000 mgr.y (mgr.14152) 287 : cephadm 1 Adjusting osd_memory_target on smithi203 to 5755M 2024-04-13T07:27:59.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:58 smithi089 bash[31522]: audit 2024-04-13T07:27:57.559260+0000 mon.a (mon.0) 575 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:00.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:27:59 smithi203 bash[23480]: cluster 2024-04-13T07:27:57.913855+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:00.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:27:59 smithi089 bash[21353]: cluster 2024-04-13T07:27:57.913855+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:00.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:27:59 smithi089 bash[31522]: cluster 2024-04-13T07:27:57.913855+0000 mgr.y (mgr.14152) 288 : cluster 0 pgmap v270: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:01.746 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:28:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:01 smithi203 bash[23480]: cluster 2024-04-13T07:27:59.914411+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:02.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:01 smithi089 bash[21353]: cluster 2024-04-13T07:27:59.914411+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:02.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:01 smithi089 bash[31522]: cluster 2024-04-13T07:27:59.914411+0000 mgr.y (mgr.14152) 289 : cluster 0 pgmap v271: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:04.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:03 smithi203 bash[23480]: cluster 2024-04-13T07:28:01.915021+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:04.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:03 smithi089 bash[21353]: cluster 2024-04-13T07:28:01.915021+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:04.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:03 smithi089 bash[31522]: cluster 2024-04-13T07:28:01.915021+0000 mgr.y (mgr.14152) 290 : cluster 0 pgmap v272: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:06.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:05 smithi203 bash[23480]: cluster 2024-04-13T07:28:03.915513+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:06.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:05 smithi203 bash[23480]: audit 2024-04-13T07:28:05.045889+0000 mon.a (mon.0) 576 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:28:06.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:05 smithi203 bash[23480]: audit 2024-04-13T07:28:05.052249+0000 mon.a (mon.0) 577 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:28:06.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:05 smithi203 bash[23480]: audit 2024-04-13T07:28:05.054072+0000 mon.a (mon.0) 578 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:06.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:05 smithi089 bash[21353]: cluster 2024-04-13T07:28:03.915513+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:06.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:05 smithi089 bash[21353]: audit 2024-04-13T07:28:05.045889+0000 mon.a (mon.0) 576 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:28:06.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:05 smithi089 bash[21353]: audit 2024-04-13T07:28:05.052249+0000 mon.a (mon.0) 577 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:28:06.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:05 smithi089 bash[21353]: audit 2024-04-13T07:28:05.054072+0000 mon.a (mon.0) 578 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:06.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:05 smithi089 bash[31522]: cluster 2024-04-13T07:28:03.915513+0000 mgr.y (mgr.14152) 291 : cluster 0 pgmap v273: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:06.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:05 smithi089 bash[31522]: audit 2024-04-13T07:28:05.045889+0000 mon.a (mon.0) 576 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2024-04-13T07:28:06.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:05 smithi089 bash[31522]: audit 2024-04-13T07:28:05.052249+0000 mon.a (mon.0) 577 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2024-04-13T07:28:06.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:05 smithi089 bash[31522]: audit 2024-04-13T07:28:05.054072+0000 mon.a (mon.0) 578 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:07.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:06 smithi203 bash[23480]: audit 2024-04-13T07:28:05.039027+0000 mgr.y (mgr.14152) 292 : audit 0 from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:28:07.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:06 smithi089 bash[21353]: audit 2024-04-13T07:28:05.039027+0000 mgr.y (mgr.14152) 292 : audit 0 from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:28:07.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:06 smithi089 bash[31522]: audit 2024-04-13T07:28:05.039027+0000 mgr.y (mgr.14152) 292 : audit 0 from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi203:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:28:08.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:07 smithi203 bash[23480]: cluster 2024-04-13T07:28:05.916059+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:08.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:07 smithi089 bash[21353]: cluster 2024-04-13T07:28:05.916059+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:08.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:07 smithi089 bash[31522]: cluster 2024-04-13T07:28:05.916059+0000 mgr.y (mgr.14152) 293 : cluster 0 pgmap v274: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:09.918 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:09 smithi203 bash[23480]: cluster 2024-04-13T07:28:07.916653+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:10.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:09 smithi089 bash[31522]: cluster 2024-04-13T07:28:07.916653+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:10.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:09 smithi089 bash[21353]: cluster 2024-04-13T07:28:07.916653+0000 mgr.y (mgr.14152) 294 : cluster 0 pgmap v275: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:12.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:11 smithi203 bash[23480]: cluster 2024-04-13T07:28:09.917207+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:12.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:11 smithi089 bash[21353]: cluster 2024-04-13T07:28:09.917207+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:12.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:11 smithi089 bash[31522]: cluster 2024-04-13T07:28:09.917207+0000 mgr.y (mgr.14152) 295 : cluster 0 pgmap v276: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:14.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:13 smithi203 bash[23480]: cluster 2024-04-13T07:28:11.917772+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:14.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:13 smithi089 bash[21353]: cluster 2024-04-13T07:28:11.917772+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:14.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:13 smithi089 bash[31522]: cluster 2024-04-13T07:28:11.917772+0000 mgr.y (mgr.14152) 296 : cluster 0 pgmap v277: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:15.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:14 smithi203 bash[23480]: audit 2024-04-13T07:28:13.613715+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.203:0/3760482320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]: dispatch 2024-04-13T07:28:15.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:14 smithi203 bash[23480]: audit 2024-04-13T07:28:13.614767+0000 mon.a (mon.0) 579 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]: dispatch 2024-04-13T07:28:15.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:14 smithi203 bash[23480]: audit 2024-04-13T07:28:13.624098+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]': finished 2024-04-13T07:28:15.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:14 smithi203 bash[23480]: cluster 2024-04-13T07:28:13.624315+0000 mon.a (mon.0) 581 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-04-13T07:28:15.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:14 smithi203 bash[23480]: audit 2024-04-13T07:28:13.624616+0000 mon.a (mon.0) 582 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:15.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:14 smithi203 bash[23480]: audit 2024-04-13T07:28:14.394516+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.203:0/622524891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[21353]: audit 2024-04-13T07:28:13.613715+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.203:0/3760482320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]: dispatch 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[21353]: audit 2024-04-13T07:28:13.614767+0000 mon.a (mon.0) 579 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]: dispatch 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[21353]: audit 2024-04-13T07:28:13.624098+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]': finished 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[21353]: cluster 2024-04-13T07:28:13.624315+0000 mon.a (mon.0) 581 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[21353]: audit 2024-04-13T07:28:13.624616+0000 mon.a (mon.0) 582 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[21353]: audit 2024-04-13T07:28:14.394516+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.203:0/622524891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[31522]: audit 2024-04-13T07:28:13.613715+0000 mon.b (mon.1) 21 : audit 1 from='client.? 172.21.15.203:0/3760482320' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]: dispatch 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[31522]: audit 2024-04-13T07:28:13.614767+0000 mon.a (mon.0) 579 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]: dispatch 2024-04-13T07:28:15.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[31522]: audit 2024-04-13T07:28:13.624098+0000 mon.a (mon.0) 580 : audit 1 from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "aa9fb5c4-06e4-45ca-85f8-f2a96be15086"}]': finished 2024-04-13T07:28:15.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[31522]: cluster 2024-04-13T07:28:13.624315+0000 mon.a (mon.0) 581 : cluster 0 osdmap e52: 8 total, 7 up, 8 in 2024-04-13T07:28:15.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[31522]: audit 2024-04-13T07:28:13.624616+0000 mon.a (mon.0) 582 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:15.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:14 smithi089 bash[31522]: audit 2024-04-13T07:28:14.394516+0000 mon.b (mon.1) 22 : audit 0 from='client.? 172.21.15.203:0/622524891' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2024-04-13T07:28:16.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:15 smithi203 bash[23480]: cluster 2024-04-13T07:28:13.918332+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:16.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:15 smithi089 bash[21353]: cluster 2024-04-13T07:28:13.918332+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:16.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:15 smithi089 bash[31522]: cluster 2024-04-13T07:28:13.918332+0000 mgr.y (mgr.14152) 297 : cluster 0 pgmap v279: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:18.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:17 smithi203 bash[23480]: cluster 2024-04-13T07:28:15.918973+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:18.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:17 smithi089 bash[21353]: cluster 2024-04-13T07:28:15.918973+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:18.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:17 smithi089 bash[31522]: cluster 2024-04-13T07:28:15.918973+0000 mgr.y (mgr.14152) 298 : cluster 0 pgmap v280: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:20.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:19 smithi203 bash[23480]: cluster 2024-04-13T07:28:17.919687+0000 mgr.y (mgr.14152) 299 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:20.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:19 smithi089 bash[21353]: cluster 2024-04-13T07:28:17.919687+0000 mgr.y (mgr.14152) 299 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:20.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:19 smithi089 bash[31522]: cluster 2024-04-13T07:28:17.919687+0000 mgr.y (mgr.14152) 299 : cluster 0 pgmap v281: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:22.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:21 smithi203 bash[23480]: cluster 2024-04-13T07:28:19.920365+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:22.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:21 smithi089 bash[21353]: cluster 2024-04-13T07:28:19.920365+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:22.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:21 smithi089 bash[31522]: cluster 2024-04-13T07:28:19.920365+0000 mgr.y (mgr.14152) 300 : cluster 0 pgmap v282: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:23.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:23 smithi203 bash[23480]: cluster 2024-04-13T07:28:21.920963+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:24.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:23 smithi089 bash[21353]: cluster 2024-04-13T07:28:21.920963+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:23 smithi089 bash[31522]: cluster 2024-04-13T07:28:21.920963+0000 mgr.y (mgr.14152) 301 : cluster 0 pgmap v283: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:26.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:25 smithi203 bash[23480]: cluster 2024-04-13T07:28:23.921532+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:26.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:25 smithi089 bash[21353]: cluster 2024-04-13T07:28:23.921532+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:25 smithi089 bash[31522]: cluster 2024-04-13T07:28:23.921532+0000 mgr.y (mgr.14152) 302 : cluster 0 pgmap v284: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:27.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:26 smithi203 bash[23480]: audit 2024-04-13T07:28:26.384745+0000 mon.a (mon.0) 583 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-13T07:28:27.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:26 smithi203 bash[23480]: audit 2024-04-13T07:28:26.386395+0000 mon.a (mon.0) 584 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:27.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:26 smithi089 bash[31522]: audit 2024-04-13T07:28:26.384745+0000 mon.a (mon.0) 583 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-13T07:28:27.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:26 smithi089 bash[31522]: audit 2024-04-13T07:28:26.386395+0000 mon.a (mon.0) 584 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:27.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:26 smithi089 bash[21353]: audit 2024-04-13T07:28:26.384745+0000 mon.a (mon.0) 583 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-13T07:28:27.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:26 smithi089 bash[21353]: audit 2024-04-13T07:28:26.386395+0000 mon.a (mon.0) 584 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:27.952 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:27 smithi203 bash[23480]: cluster 2024-04-13T07:28:25.922093+0000 mgr.y (mgr.14152) 303 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:27.952 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:27 smithi203 bash[23480]: cephadm 2024-04-13T07:28:26.387908+0000 mgr.y (mgr.14152) 304 : cephadm 1 Deploying daemon osd.7 on smithi203 2024-04-13T07:28:28.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:27 smithi089 bash[31522]: cluster 2024-04-13T07:28:25.922093+0000 mgr.y (mgr.14152) 303 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:28.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:27 smithi089 bash[31522]: cephadm 2024-04-13T07:28:26.387908+0000 mgr.y (mgr.14152) 304 : cephadm 1 Deploying daemon osd.7 on smithi203 2024-04-13T07:28:28.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:27 smithi089 bash[21353]: cluster 2024-04-13T07:28:25.922093+0000 mgr.y (mgr.14152) 303 : cluster 0 pgmap v285: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:28.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:27 smithi089 bash[21353]: cephadm 2024-04-13T07:28:26.387908+0000 mgr.y (mgr.14152) 304 : cephadm 1 Deploying daemon osd.7 on smithi203 2024-04-13T07:28:29.917 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:29 smithi203 bash[23480]: cluster 2024-04-13T07:28:27.922690+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:30.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:29 smithi089 bash[21353]: cluster 2024-04-13T07:28:27.922690+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:29 smithi089 bash[31522]: cluster 2024-04-13T07:28:27.922690+0000 mgr.y (mgr.14152) 305 : cluster 0 pgmap v286: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:30.858 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:30.858 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:30.858 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:30.859 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:30.859 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:31.125 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:31.126 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:31.126 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:31.126 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:31.127 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:28:30 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:28:32.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:31 smithi203 bash[23480]: cluster 2024-04-13T07:28:29.923279+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:32.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:31 smithi203 bash[23480]: audit 2024-04-13T07:28:31.127633+0000 mon.a (mon.0) 585 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:32.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:31 smithi203 bash[23480]: audit 2024-04-13T07:28:31.137582+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:32.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:31 smithi203 bash[23480]: audit 2024-04-13T07:28:31.141112+0000 mon.a (mon.0) 587 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:28:32.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:31 smithi089 bash[31522]: cluster 2024-04-13T07:28:29.923279+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:32.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:31 smithi089 bash[31522]: audit 2024-04-13T07:28:31.127633+0000 mon.a (mon.0) 585 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:32.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:31 smithi089 bash[31522]: audit 2024-04-13T07:28:31.137582+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:32.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:31 smithi089 bash[31522]: audit 2024-04-13T07:28:31.141112+0000 mon.a (mon.0) 587 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:28:32.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:31 smithi089 bash[21353]: cluster 2024-04-13T07:28:29.923279+0000 mgr.y (mgr.14152) 306 : cluster 0 pgmap v287: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:32.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:31 smithi089 bash[21353]: audit 2024-04-13T07:28:31.127633+0000 mon.a (mon.0) 585 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:32.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:31 smithi089 bash[21353]: audit 2024-04-13T07:28:31.137582+0000 mon.a (mon.0) 586 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:32.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:31 smithi089 bash[21353]: audit 2024-04-13T07:28:31.141112+0000 mon.a (mon.0) 587 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:28:34.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:33 smithi203 bash[23480]: cluster 2024-04-13T07:28:31.923886+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:34.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:33 smithi089 bash[31522]: cluster 2024-04-13T07:28:31.923886+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:34.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:33 smithi089 bash[21353]: cluster 2024-04-13T07:28:31.923886+0000 mgr.y (mgr.14152) 307 : cluster 0 pgmap v288: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:36.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:35 smithi203 bash[23480]: cluster 2024-04-13T07:28:33.924559+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:36.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:35 smithi089 bash[31522]: cluster 2024-04-13T07:28:33.924559+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:36.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:35 smithi089 bash[21353]: cluster 2024-04-13T07:28:33.924559+0000 mgr.y (mgr.14152) 308 : cluster 0 pgmap v289: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:37.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:37 smithi203 bash[23480]: cluster 2024-04-13T07:28:35.925149+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:37.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:37 smithi203 bash[23480]: audit 2024-04-13T07:28:36.541517+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:37.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:37 smithi203 bash[23480]: audit 2024-04-13T07:28:36.551460+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:37.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:37 smithi203 bash[23480]: audit 2024-04-13T07:28:37.079332+0000 mon.a (mon.0) 590 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:37.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:37 smithi203 bash[23480]: audit 2024-04-13T07:28:37.081064+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:28:37.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:37 smithi203 bash[23480]: audit 2024-04-13T07:28:37.091698+0000 mon.a (mon.0) 592 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:37.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[31522]: cluster 2024-04-13T07:28:35.925149+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:37.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[31522]: audit 2024-04-13T07:28:36.541517+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:37.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[31522]: audit 2024-04-13T07:28:36.551460+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:37.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[31522]: audit 2024-04-13T07:28:37.079332+0000 mon.a (mon.0) 590 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:37.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[31522]: audit 2024-04-13T07:28:37.081064+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:28:37.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[31522]: audit 2024-04-13T07:28:37.091698+0000 mon.a (mon.0) 592 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:37.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[21353]: cluster 2024-04-13T07:28:35.925149+0000 mgr.y (mgr.14152) 309 : cluster 0 pgmap v290: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:37.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[21353]: audit 2024-04-13T07:28:36.541517+0000 mon.a (mon.0) 588 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:37.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[21353]: audit 2024-04-13T07:28:36.551460+0000 mon.a (mon.0) 589 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:37.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[21353]: audit 2024-04-13T07:28:37.079332+0000 mon.a (mon.0) 590 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:37.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[21353]: audit 2024-04-13T07:28:37.081064+0000 mon.a (mon.0) 591 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:28:37.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:37 smithi089 bash[21353]: audit 2024-04-13T07:28:37.091698+0000 mon.a (mon.0) 592 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:38.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:38 smithi089 bash[21353]: audit 2024-04-13T07:28:37.856998+0000 mon.b (mon.1) 23 : audit 1 from='osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-13T07:28:38.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:38 smithi089 bash[21353]: audit 2024-04-13T07:28:37.858068+0000 mon.a (mon.0) 593 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-13T07:28:38.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:38 smithi089 bash[31522]: audit 2024-04-13T07:28:37.856998+0000 mon.b (mon.1) 23 : audit 1 from='osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-13T07:28:38.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:38 smithi089 bash[31522]: audit 2024-04-13T07:28:37.858068+0000 mon.a (mon.0) 593 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-13T07:28:39.006 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:38 smithi203 bash[23480]: audit 2024-04-13T07:28:37.856998+0000 mon.b (mon.1) 23 : audit 1 from='osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-13T07:28:39.006 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:38 smithi203 bash[23480]: audit 2024-04-13T07:28:37.858068+0000 mon.a (mon.0) 593 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2024-04-13T07:28:39.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:39 smithi203 bash[23480]: cluster 2024-04-13T07:28:37.925720+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:39.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:39 smithi203 bash[23480]: audit 2024-04-13T07:28:38.557349+0000 mon.a (mon.0) 594 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-13T07:28:39.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:39 smithi203 bash[23480]: cluster 2024-04-13T07:28:38.557502+0000 mon.a (mon.0) 595 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-04-13T07:28:39.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:39 smithi203 bash[23480]: audit 2024-04-13T07:28:38.557786+0000 mon.a (mon.0) 596 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:39.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:39 smithi203 bash[23480]: audit 2024-04-13T07:28:38.558316+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:28:39.816 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:39 smithi203 bash[23480]: audit 2024-04-13T07:28:38.559397+0000 mon.a (mon.0) 597 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:28:40.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[21353]: cluster 2024-04-13T07:28:37.925720+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[21353]: audit 2024-04-13T07:28:38.557349+0000 mon.a (mon.0) 594 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[21353]: cluster 2024-04-13T07:28:38.557502+0000 mon.a (mon.0) 595 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[21353]: audit 2024-04-13T07:28:38.557786+0000 mon.a (mon.0) 596 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[21353]: audit 2024-04-13T07:28:38.558316+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[21353]: audit 2024-04-13T07:28:38.559397+0000 mon.a (mon.0) 597 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[31522]: cluster 2024-04-13T07:28:37.925720+0000 mgr.y (mgr.14152) 310 : cluster 0 pgmap v291: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[31522]: audit 2024-04-13T07:28:38.557349+0000 mon.a (mon.0) 594 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[31522]: cluster 2024-04-13T07:28:38.557502+0000 mon.a (mon.0) 595 : cluster 0 osdmap e53: 8 total, 7 up, 8 in 2024-04-13T07:28:40.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[31522]: audit 2024-04-13T07:28:38.557786+0000 mon.a (mon.0) 596 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:40.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[31522]: audit 2024-04-13T07:28:38.558316+0000 mon.b (mon.1) 24 : audit 1 from='osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:28:40.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:39 smithi089 bash[31522]: audit 2024-04-13T07:28:38.559397+0000 mon.a (mon.0) 597 : audit 1 from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:28:40.440 INFO:teuthology.orchestra.run.smithi203.stdout:Created osd(s) 7 on host 'smithi203' 2024-04-13T07:28:40.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:40 smithi203 bash[23480]: audit 2024-04-13T07:28:39.557878+0000 mon.a (mon.0) 598 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:28:40.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:40 smithi203 bash[23480]: cluster 2024-04-13T07:28:39.558071+0000 mon.a (mon.0) 599 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-04-13T07:28:40.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:40 smithi203 bash[23480]: audit 2024-04-13T07:28:39.558712+0000 mon.a (mon.0) 600 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:40.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:40 smithi203 bash[23480]: audit 2024-04-13T07:28:39.566900+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:40.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:40 smithi203 bash[23480]: audit 2024-04-13T07:28:40.079531+0000 mon.a (mon.0) 602 : audit 1 from='osd.7 ' entity='osd.7' 2024-04-13T07:28:40.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:40 smithi203 bash[23480]: audit 2024-04-13T07:28:40.402391+0000 mon.a (mon.0) 603 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:40.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:40 smithi203 bash[23480]: audit 2024-04-13T07:28:40.412663+0000 mon.a (mon.0) 604 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:40.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:40 smithi203 bash[23480]: audit 2024-04-13T07:28:40.434688+0000 mon.a (mon.0) 605 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:28:41.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[21353]: audit 2024-04-13T07:28:39.557878+0000 mon.a (mon.0) 598 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[21353]: cluster 2024-04-13T07:28:39.558071+0000 mon.a (mon.0) 599 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[21353]: audit 2024-04-13T07:28:39.558712+0000 mon.a (mon.0) 600 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[21353]: audit 2024-04-13T07:28:39.566900+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[21353]: audit 2024-04-13T07:28:40.079531+0000 mon.a (mon.0) 602 : audit 1 from='osd.7 ' entity='osd.7' 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[21353]: audit 2024-04-13T07:28:40.402391+0000 mon.a (mon.0) 603 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[21353]: audit 2024-04-13T07:28:40.412663+0000 mon.a (mon.0) 604 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[21353]: audit 2024-04-13T07:28:40.434688+0000 mon.a (mon.0) 605 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[31522]: audit 2024-04-13T07:28:39.557878+0000 mon.a (mon.0) 598 : audit 1 from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]': finished 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[31522]: cluster 2024-04-13T07:28:39.558071+0000 mon.a (mon.0) 599 : cluster 0 osdmap e54: 8 total, 7 up, 8 in 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[31522]: audit 2024-04-13T07:28:39.558712+0000 mon.a (mon.0) 600 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[31522]: audit 2024-04-13T07:28:39.566900+0000 mon.a (mon.0) 601 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:41.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[31522]: audit 2024-04-13T07:28:40.079531+0000 mon.a (mon.0) 602 : audit 1 from='osd.7 ' entity='osd.7' 2024-04-13T07:28:41.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[31522]: audit 2024-04-13T07:28:40.402391+0000 mon.a (mon.0) 603 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:41.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[31522]: audit 2024-04-13T07:28:40.412663+0000 mon.a (mon.0) 604 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:41.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:40 smithi089 bash[31522]: audit 2024-04-13T07:28:40.434688+0000 mon.a (mon.0) 605 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:28:41.270 DEBUG:teuthology.orchestra.run.smithi203:osd.7> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.7.service 2024-04-13T07:28:41.273 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2024-04-13T07:28:41.273 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd stat -f json 2024-04-13T07:28:42.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:41 smithi203 bash[23480]: cluster 2024-04-13T07:28:38.869129+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:28:42.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:41 smithi203 bash[23480]: cluster 2024-04-13T07:28:38.869259+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:28:42.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:41 smithi203 bash[23480]: cluster 2024-04-13T07:28:39.926253+0000 mgr.y (mgr.14152) 311 : cluster 0 pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:42.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:41 smithi203 bash[23480]: cluster 2024-04-13T07:28:40.560392+0000 mon.a (mon.0) 606 : cluster 1 osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771] boot 2024-04-13T07:28:42.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:41 smithi203 bash[23480]: cluster 2024-04-13T07:28:40.560685+0000 mon.a (mon.0) 607 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-04-13T07:28:42.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:41 smithi203 bash[23480]: audit 2024-04-13T07:28:40.560980+0000 mon.a (mon.0) 608 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:42.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:41 smithi203 bash[23480]: cluster 2024-04-13T07:28:41.562155+0000 mon.a (mon.0) 609 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-04-13T07:28:42.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[21353]: cluster 2024-04-13T07:28:38.869129+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:28:42.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[21353]: cluster 2024-04-13T07:28:38.869259+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:28:42.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[21353]: cluster 2024-04-13T07:28:39.926253+0000 mgr.y (mgr.14152) 311 : cluster 0 pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:42.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[21353]: cluster 2024-04-13T07:28:40.560392+0000 mon.a (mon.0) 606 : cluster 1 osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771] boot 2024-04-13T07:28:42.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[21353]: cluster 2024-04-13T07:28:40.560685+0000 mon.a (mon.0) 607 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-04-13T07:28:42.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[21353]: audit 2024-04-13T07:28:40.560980+0000 mon.a (mon.0) 608 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:42.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[21353]: cluster 2024-04-13T07:28:41.562155+0000 mon.a (mon.0) 609 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-04-13T07:28:42.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[31522]: cluster 2024-04-13T07:28:38.869129+0000 osd.7 (osd.7) 1 : cluster 0 purged_snaps scrub starts 2024-04-13T07:28:42.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[31522]: cluster 2024-04-13T07:28:38.869259+0000 osd.7 (osd.7) 2 : cluster 0 purged_snaps scrub ok 2024-04-13T07:28:42.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[31522]: cluster 2024-04-13T07:28:39.926253+0000 mgr.y (mgr.14152) 311 : cluster 0 pgmap v294: 1 pgs: 1 active+clean; 577 KiB data, 198 MiB used, 626 GiB / 626 GiB avail 2024-04-13T07:28:42.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[31522]: cluster 2024-04-13T07:28:40.560392+0000 mon.a (mon.0) 606 : cluster 1 osd.7 [v2:172.21.15.203:6824/3848298771,v1:172.21.15.203:6825/3848298771] boot 2024-04-13T07:28:42.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[31522]: cluster 2024-04-13T07:28:40.560685+0000 mon.a (mon.0) 607 : cluster 0 osdmap e55: 8 total, 8 up, 8 in 2024-04-13T07:28:42.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[31522]: audit 2024-04-13T07:28:40.560980+0000 mon.a (mon.0) 608 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:28:42.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:41 smithi089 bash[31522]: cluster 2024-04-13T07:28:41.562155+0000 mon.a (mon.0) 609 : cluster 0 osdmap e56: 8 total, 8 up, 8 in 2024-04-13T07:28:44.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:43 smithi203 bash[23480]: cluster 2024-04-13T07:28:41.926813+0000 mgr.y (mgr.14152) 312 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 516 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:28:44.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:43 smithi203 bash[23480]: cluster 2024-04-13T07:28:42.565282+0000 mon.a (mon.0) 610 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-04-13T07:28:44.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:43 smithi203 bash[23480]: cluster 2024-04-13T07:28:42.829920+0000 mon.a (mon.0) 611 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-04-13T07:28:44.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:43 smithi089 bash[21353]: cluster 2024-04-13T07:28:41.926813+0000 mgr.y (mgr.14152) 312 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 516 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:28:44.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:43 smithi089 bash[21353]: cluster 2024-04-13T07:28:42.565282+0000 mon.a (mon.0) 610 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-04-13T07:28:44.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:43 smithi089 bash[21353]: cluster 2024-04-13T07:28:42.829920+0000 mon.a (mon.0) 611 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-04-13T07:28:44.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:43 smithi089 bash[31522]: cluster 2024-04-13T07:28:41.926813+0000 mgr.y (mgr.14152) 312 : cluster 0 pgmap v297: 1 pgs: 1 active+clean; 577 KiB data, 516 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:28:44.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:43 smithi089 bash[31522]: cluster 2024-04-13T07:28:42.565282+0000 mon.a (mon.0) 610 : cluster 0 osdmap e57: 8 total, 8 up, 8 in 2024-04-13T07:28:44.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:43 smithi089 bash[31522]: cluster 2024-04-13T07:28:42.829920+0000 mon.a (mon.0) 611 : cluster 0 osdmap e58: 8 total, 8 up, 8 in 2024-04-13T07:28:45.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:44 smithi203 bash[23480]: cluster 2024-04-13T07:28:43.830793+0000 mon.a (mon.0) 612 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-04-13T07:28:45.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:44 smithi089 bash[21353]: cluster 2024-04-13T07:28:43.830793+0000 mon.a (mon.0) 612 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-04-13T07:28:45.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:44 smithi089 bash[31522]: cluster 2024-04-13T07:28:43.830793+0000 mon.a (mon.0) 612 : cluster 0 osdmap e59: 8 total, 8 up, 8 in 2024-04-13T07:28:45.983 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:28:46.236 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:45 smithi203 bash[23480]: cluster 2024-04-13T07:28:43.927373+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 516 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:28:46.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:45 smithi089 bash[21353]: cluster 2024-04-13T07:28:43.927373+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 516 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:28:46.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:45 smithi089 bash[31522]: cluster 2024-04-13T07:28:43.927373+0000 mgr.y (mgr.14152) 313 : cluster 0 pgmap v301: 1 pgs: 1 active+clean; 577 KiB data, 516 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:28:48.131 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:47 smithi089 bash[21353]: cluster 2024-04-13T07:28:45.927983+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s, 0 objects/s recovering 2024-04-13T07:28:48.131 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:47 smithi089 bash[31522]: cluster 2024-04-13T07:28:45.927983+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s, 0 objects/s recovering 2024-04-13T07:28:48.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:47 smithi203 bash[23480]: cluster 2024-04-13T07:28:45.927983+0000 mgr.y (mgr.14152) 314 : cluster 0 pgmap v302: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 132 KiB/s, 0 objects/s recovering 2024-04-13T07:28:48.529 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:28:49.105 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:48 smithi089 bash[21353]: audit 2024-04-13T07:28:48.534068+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.89:0/1043139408' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-13T07:28:49.105 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:48 smithi089 bash[31522]: audit 2024-04-13T07:28:48.534068+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.89:0/1043139408' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-13T07:28:49.106 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":59,"num_osds":8,"num_up_osds":8,"osd_up_since":1712993320,"num_in_osds":8,"osd_in_since":1712993293,"num_remapped_pgs":0} 2024-04-13T07:28:49.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-13T07:28:49.145 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:48 smithi203 bash[23480]: audit 2024-04-13T07:28:48.534068+0000 mon.c (mon.2) 13 : audit 0 from='client.? 172.21.15.89:0/1043139408' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2024-04-13T07:28:50.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:49 smithi203 bash[23480]: cluster 2024-04-13T07:28:47.928585+0000 mgr.y (mgr.14152) 315 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:28:50.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:49 smithi089 bash[31522]: cluster 2024-04-13T07:28:47.928585+0000 mgr.y (mgr.14152) 315 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:28:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:49 smithi089 bash[21353]: cluster 2024-04-13T07:28:47.928585+0000 mgr.y (mgr.14152) 315 : cluster 0 pgmap v303: 1 pgs: 1 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2024-04-13T07:28:50.934 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: cephadm 2024-04-13T07:28:49.926332+0000 mgr.y (mgr.14152) 316 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: cluster 2024-04-13T07:28:49.929256+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.936435+0000 mon.a (mon.0) 613 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: cephadm 2024-04-13T07:28:49.926332+0000 mgr.y (mgr.14152) 316 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: cluster 2024-04-13T07:28:49.929256+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.936435+0000 mon.a (mon.0) 613 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.946523+0000 mon.a (mon.0) 614 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.949204+0000 mon.a (mon.0) 615 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.950943+0000 mon.a (mon.0) 616 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.952708+0000 mon.a (mon.0) 617 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.954494+0000 mon.a (mon.0) 618 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: cephadm 2024-04-13T07:28:49.955871+0000 mgr.y (mgr.14152) 318 : cephadm 1 Adjusting osd_memory_target on smithi203 to 4316M 2024-04-13T07:28:50.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.964518+0000 mon.a (mon.0) 619 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:50.936 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.966582+0000 mon.a (mon.0) 620 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:50.936 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.968250+0000 mon.a (mon.0) 621 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:28:50.936 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[21353]: audit 2024-04-13T07:28:49.978209+0000 mon.a (mon.0) 622 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: cephadm 2024-04-13T07:28:49.926332+0000 mgr.y (mgr.14152) 316 : cephadm 1 Detected new or changed devices on smithi203 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: cluster 2024-04-13T07:28:49.929256+0000 mgr.y (mgr.14152) 317 : cluster 0 pgmap v304: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 78 KiB/s, 0 objects/s recovering 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.936435+0000 mon.a (mon.0) 613 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.946523+0000 mon.a (mon.0) 614 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.949204+0000 mon.a (mon.0) 615 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.950943+0000 mon.a (mon.0) 616 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.952708+0000 mon.a (mon.0) 617 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.954494+0000 mon.a (mon.0) 618 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: cephadm 2024-04-13T07:28:49.955871+0000 mgr.y (mgr.14152) 318 : cephadm 1 Adjusting osd_memory_target on smithi203 to 4316M 2024-04-13T07:28:51.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.964518+0000 mon.a (mon.0) 619 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:51.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.966582+0000 mon.a (mon.0) 620 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:51.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.968250+0000 mon.a (mon.0) 621 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:28:51.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:50 smithi203 bash[23480]: audit 2024-04-13T07:28:49.978209+0000 mon.a (mon.0) 622 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:51.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.946523+0000 mon.a (mon.0) 614 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:51.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.949204+0000 mon.a (mon.0) 615 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:51.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.950943+0000 mon.a (mon.0) 616 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:51.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.952708+0000 mon.a (mon.0) 617 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:51.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.954494+0000 mon.a (mon.0) 618 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:28:51.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: cephadm 2024-04-13T07:28:49.955871+0000 mgr.y (mgr.14152) 318 : cephadm 1 Adjusting osd_memory_target on smithi203 to 4316M 2024-04-13T07:28:51.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.964518+0000 mon.a (mon.0) 619 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:51.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.966582+0000 mon.a (mon.0) 620 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:28:51.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.968250+0000 mon.a (mon.0) 621 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:28:51.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:50 smithi089 bash[31522]: audit 2024-04-13T07:28:49.978209+0000 mon.a (mon.0) 622 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:28:53.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:52 smithi203 bash[23480]: cluster 2024-04-13T07:28:51.929873+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-13T07:28:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:52 smithi089 bash[21353]: cluster 2024-04-13T07:28:51.929873+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-13T07:28:53.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:52 smithi089 bash[31522]: cluster 2024-04-13T07:28:51.929873+0000 mgr.y (mgr.14152) 319 : cluster 0 pgmap v305: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 63 KiB/s, 0 objects/s recovering 2024-04-13T07:28:53.811 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:28:55.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:54 smithi203 bash[23480]: cluster 2024-04-13T07:28:53.930475+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-13T07:28:55.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:54 smithi089 bash[21353]: cluster 2024-04-13T07:28:53.930475+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-13T07:28:55.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:54 smithi089 bash[31522]: cluster 2024-04-13T07:28:53.930475+0000 mgr.y (mgr.14152) 320 : cluster 0 pgmap v306: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 57 KiB/s, 0 objects/s recovering 2024-04-13T07:28:56.411 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:28:56.411 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":59,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","created":"2024-04-13T07:19:16.936150+0000","modified":"2024-04-13T07:28:43.819460+0000","last_up_change":"2024-04-13T07:28:40.551327+0000","last_in_change":"2024-04-13T07:28:13.615937+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-13T07:24:24.430140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9de691f5-25ed-472d-b6dd-dc3908fa85c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6803","nonce":3872923960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6805","nonce":3872923960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6809","nonce":3872923960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6807","nonce":3872923960}]},"public_addr":"172.21.15.89:6803/3872923960","cluster_addr":"172.21.15.89:6805/3872923960","heartbeat_back_addr":"172.21.15.89:6809/3872923960","heartbeat_front_addr":"172.21.15.89:6807/3872923960","state":["exists","up"]},{"osd":1,"uuid":"ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6811","nonce":302139657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6813","nonce":302139657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6817","nonce":302139657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6815","nonce":302139657}]},"public_addr":"172.21.15.89:6811/302139657","cluster_addr":"172.21.15.89:6813/302139657","heartbeat_back_addr":"172.21.15.89:6817/302139657","heartbeat_front_addr":"172.21.15.89:6815/302139657","state":["exists","up"]},{"osd":2,"uuid":"f86d6aea-97d3-4d75-99d0-6e5781d7a990","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6819","nonce":46732397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6821","nonce":46732397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6825","nonce":46732397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6823","nonce":46732397}]},"public_addr":"172.21.15.89:6819/46732397","cluster_addr":"172.21.15.89:6821/46732397","heartbeat_back_addr":"172.21.15.89:6825/46732397","heartbeat_front_addr":"172.21.15.89:6823/46732397","state":["exists","up"]},{"osd":3,"uuid":"37c263d3-d642-4cfa-99da-5a33e1bef16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6827","nonce":3306593060}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6829","nonce":3306593060}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6833","nonce":3306593060}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6831","nonce":3306593060}]},"public_addr":"172.21.15.89:6827/3306593060","cluster_addr":"172.21.15.89:6829/3306593060","heartbeat_back_addr":"172.21.15.89:6833/3306593060","heartbeat_front_addr":"172.21.15.89:6831/3306593060","state":["exists","up"]},{"osd":4,"uuid":"c1befaa2-c9f5-4023-9909-0577509ce44d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6801","nonce":3982510878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6803","nonce":3982510878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6807","nonce":3982510878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6805","nonce":3982510878}]},"public_addr":"172.21.15.203:6801/3982510878","cluster_addr":"172.21.15.203:6803/3982510878","heartbeat_back_addr":"172.21.15.203:6807/3982510878","heartbeat_front_addr":"172.21.15.203:6805/3982510878","state":["exists","up"]},{"osd":5,"uuid":"b45987d4-fc0f-45ad-9f0d-49240d898bfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6809","nonce":209623809}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6811","nonce":209623809}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6815","nonce":209623809}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6813","nonce":209623809}]},"public_addr":"172.21.15.203:6809/209623809","cluster_addr":"172.21.15.203:6811/209623809","heartbeat_back_addr":"172.21.15.203:6815/209623809","heartbeat_front_addr":"172.21.15.203:6813/209623809","state":["exists","up"]},{"osd":6,"uuid":"91bbf36b-799c-49df-8fbf-42a958699642","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6817","nonce":2138665907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6819","nonce":2138665907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6823","nonce":2138665907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6821","nonce":2138665907}]},"public_addr":"172.21.15.203:6817/2138665907","cluster_addr":"172.21.15.203:6819/2138665907","heartbeat_back_addr":"172.21.15.203:6823/2138665907","heartbeat_front_addr":"172.21.15.203:6821/2138665907","state":["exists","up"]},{"osd":7,"uuid":"aa9fb5c4-06e4-45ca-85f8-f2a96be15086","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6825","nonce":3848298771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6827","nonce":3848298771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6831","nonce":3848298771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6829","nonce":3848298771}]},"public_addr":"172.21.15.203:6825/3848298771","cluster_addr":"172.21.15.203:6827/3848298771","heartbeat_back_addr":"172.21.15.203:6831/3848298771","heartbeat_front_addr":"172.21.15.203:6829/3848298771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:22:41.252314+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:23:31.382195+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:24:20.791478+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:10.117632+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:59.130696+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:26:51.940707+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:27:44.966139+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:28:38.869265+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:6800/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6801/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/28202064":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/2936084513":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/808783885":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/868096906":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2680580613":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6801/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/1215893273":"2024-04-14T07:20:19.796707+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-13T07:28:57.017 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2024-04-13T07:24:24.430140+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}, 'read_balance': {'score_acting': 7.889999866485596, 'score_stable': 7.889999866485596, 'optimal_score': 0.3799999952316284, 'raw_score_acting': 3, 'raw_score_stable': 3, 'primary_affinity_weighted': 1, 'average_primary_affinity': 1, 'average_primary_affinity_weighted': 1}}] 2024-04-13T07:28:57.017 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd pool get .mgr pg_num 2024-04-13T07:28:57.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:56 smithi203 bash[23480]: cluster 2024-04-13T07:28:55.931087+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-13T07:28:57.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:56 smithi203 bash[23480]: audit 2024-04-13T07:28:56.414264+0000 mon.a (mon.0) 623 : audit 0 from='client.? 172.21.15.89:0/562627046' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:28:57.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:56 smithi089 bash[21353]: cluster 2024-04-13T07:28:55.931087+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-13T07:28:57.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:56 smithi089 bash[21353]: audit 2024-04-13T07:28:56.414264+0000 mon.a (mon.0) 623 : audit 0 from='client.? 172.21.15.89:0/562627046' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:28:57.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:56 smithi089 bash[31522]: cluster 2024-04-13T07:28:55.931087+0000 mgr.y (mgr.14152) 321 : cluster 0 pgmap v307: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2024-04-13T07:28:57.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:56 smithi089 bash[31522]: audit 2024-04-13T07:28:56.414264+0000 mon.a (mon.0) 623 : audit 0 from='client.? 172.21.15.89:0/562627046' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:28:59.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:28:58 smithi203 bash[23480]: cluster 2024-04-13T07:28:57.931664+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:28:59.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:28:58 smithi089 bash[21353]: cluster 2024-04-13T07:28:57.931664+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:28:59.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:28:58 smithi089 bash[31522]: cluster 2024-04-13T07:28:57.931664+0000 mgr.y (mgr.14152) 322 : cluster 0 pgmap v308: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:00.722 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:29:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:00 smithi089 bash[21353]: cluster 2024-04-13T07:28:59.932177+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:00 smithi089 bash[31522]: cluster 2024-04-13T07:28:59.932177+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:01.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:00 smithi203 bash[23480]: cluster 2024-04-13T07:28:59.932177+0000 mgr.y (mgr.14152) 323 : cluster 0 pgmap v309: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:03.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:02 smithi203 bash[23480]: cluster 2024-04-13T07:29:01.932763+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:03.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:02 smithi089 bash[21353]: cluster 2024-04-13T07:29:01.932763+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:03.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:02 smithi089 bash[31522]: cluster 2024-04-13T07:29:01.932763+0000 mgr.y (mgr.14152) 324 : cluster 0 pgmap v310: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:03.440 INFO:teuthology.orchestra.run.smithi089.stdout:pg_num: 1 2024-04-13T07:29:04.115 INFO:tasks.cephadm:Adding prometheus.a on smithi203 2024-04-13T07:29:04.115 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch apply prometheus '1;smithi203=a' 2024-04-13T07:29:04.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:03 smithi203 bash[23480]: audit 2024-04-13T07:29:03.444928+0000 mon.a (mon.0) 624 : audit 0 from='client.? 172.21.15.89:0/1484692692' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-13T07:29:04.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:03 smithi089 bash[21353]: audit 2024-04-13T07:29:03.444928+0000 mon.a (mon.0) 624 : audit 0 from='client.? 172.21.15.89:0/1484692692' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-13T07:29:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:03 smithi089 bash[31522]: audit 2024-04-13T07:29:03.444928+0000 mon.a (mon.0) 624 : audit 0 from='client.? 172.21.15.89:0/1484692692' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2024-04-13T07:29:05.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:04 smithi203 bash[23480]: cluster 2024-04-13T07:29:03.933346+0000 mgr.y (mgr.14152) 325 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:05.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:04 smithi089 bash[21353]: cluster 2024-04-13T07:29:03.933346+0000 mgr.y (mgr.14152) 325 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:05.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:04 smithi089 bash[31522]: cluster 2024-04-13T07:29:03.933346+0000 mgr.y (mgr.14152) 325 : cluster 0 pgmap v311: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:07.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:06 smithi203 bash[23480]: cluster 2024-04-13T07:29:05.933920+0000 mgr.y (mgr.14152) 326 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:07.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:06 smithi089 bash[21353]: cluster 2024-04-13T07:29:05.933920+0000 mgr.y (mgr.14152) 326 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:07.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:06 smithi089 bash[31522]: cluster 2024-04-13T07:29:05.933920+0000 mgr.y (mgr.14152) 326 : cluster 0 pgmap v312: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:08.816 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:29:09.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:08 smithi203 bash[23480]: cluster 2024-04-13T07:29:07.934487+0000 mgr.y (mgr.14152) 327 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:08 smithi089 bash[21353]: cluster 2024-04-13T07:29:07.934487+0000 mgr.y (mgr.14152) 327 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:09.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:08 smithi089 bash[31522]: cluster 2024-04-13T07:29:07.934487+0000 mgr.y (mgr.14152) 327 : cluster 0 pgmap v313: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:11.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:10 smithi203 bash[23480]: cluster 2024-04-13T07:29:09.935107+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:11.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:10 smithi089 bash[31522]: cluster 2024-04-13T07:29:09.935107+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:11.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:10 smithi089 bash[21353]: cluster 2024-04-13T07:29:09.935107+0000 mgr.y (mgr.14152) 328 : cluster 0 pgmap v314: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:11.971 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled prometheus update... 2024-04-13T07:29:12.788 DEBUG:teuthology.orchestra.run.smithi203:prometheus.a> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@prometheus.a.service 2024-04-13T07:29:12.790 INFO:tasks.cephadm:Adding node-exporter.a on smithi089 2024-04-13T07:29:12.790 INFO:tasks.cephadm:Adding node-exporter.b on smithi203 2024-04-13T07:29:12.791 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch apply node-exporter '2;smithi089=a;smithi203=b' 2024-04-13T07:29:12.968 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: cluster 2024-04-13T07:29:11.935677+0000 mgr.y (mgr.14152) 329 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:12.968 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: audit 2024-04-13T07:29:11.963355+0000 mgr.y (mgr.14152) 330 : audit 0 from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:12.968 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: cephadm 2024-04-13T07:29:11.966788+0000 mgr.y (mgr.14152) 331 : cephadm 1 Saving service prometheus spec with placement smithi203=a;count:1 2024-04-13T07:29:12.969 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: audit 2024-04-13T07:29:11.974796+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:12.969 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: audit 2024-04-13T07:29:12.018488+0000 mon.a (mon.0) 626 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:12.969 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: audit 2024-04-13T07:29:12.022079+0000 mon.a (mon.0) 627 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:29:12.969 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: audit 2024-04-13T07:29:12.023881+0000 mon.a (mon.0) 628 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:29:12.969 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: audit 2024-04-13T07:29:12.036364+0000 mon.a (mon.0) 629 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:12.969 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:12 smithi203 bash[23480]: cephadm 2024-04-13T07:29:12.330231+0000 mgr.y (mgr.14152) 332 : cephadm 1 Deploying daemon prometheus.a on smithi203 2024-04-13T07:29:13.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: cluster 2024-04-13T07:29:11.935677+0000 mgr.y (mgr.14152) 329 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:13.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: audit 2024-04-13T07:29:11.963355+0000 mgr.y (mgr.14152) 330 : audit 0 from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:13.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: cephadm 2024-04-13T07:29:11.966788+0000 mgr.y (mgr.14152) 331 : cephadm 1 Saving service prometheus spec with placement smithi203=a;count:1 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: audit 2024-04-13T07:29:11.974796+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: audit 2024-04-13T07:29:12.018488+0000 mon.a (mon.0) 626 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: audit 2024-04-13T07:29:12.022079+0000 mon.a (mon.0) 627 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: audit 2024-04-13T07:29:12.023881+0000 mon.a (mon.0) 628 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: audit 2024-04-13T07:29:12.036364+0000 mon.a (mon.0) 629 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[21353]: cephadm 2024-04-13T07:29:12.330231+0000 mgr.y (mgr.14152) 332 : cephadm 1 Deploying daemon prometheus.a on smithi203 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: cluster 2024-04-13T07:29:11.935677+0000 mgr.y (mgr.14152) 329 : cluster 0 pgmap v315: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: audit 2024-04-13T07:29:11.963355+0000 mgr.y (mgr.14152) 330 : audit 0 from='client.24274 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: cephadm 2024-04-13T07:29:11.966788+0000 mgr.y (mgr.14152) 331 : cephadm 1 Saving service prometheus spec with placement smithi203=a;count:1 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: audit 2024-04-13T07:29:11.974796+0000 mon.a (mon.0) 625 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:13.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: audit 2024-04-13T07:29:12.018488+0000 mon.a (mon.0) 626 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:13.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: audit 2024-04-13T07:29:12.022079+0000 mon.a (mon.0) 627 : audit 0 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:29:13.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: audit 2024-04-13T07:29:12.023881+0000 mon.a (mon.0) 628 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:29:13.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: audit 2024-04-13T07:29:12.036364+0000 mon.a (mon.0) 629 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:13.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:12 smithi089 bash[31522]: cephadm 2024-04-13T07:29:12.330231+0000 mgr.y (mgr.14152) 332 : cephadm 1 Deploying daemon prometheus.a on smithi203 2024-04-13T07:29:15.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:14 smithi203 bash[23480]: cluster 2024-04-13T07:29:13.936254+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:15.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:14 smithi089 bash[21353]: cluster 2024-04-13T07:29:13.936254+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:15.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:14 smithi089 bash[31522]: cluster 2024-04-13T07:29:13.936254+0000 mgr.y (mgr.14152) 333 : cluster 0 pgmap v316: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:17.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:16 smithi203 bash[23480]: cluster 2024-04-13T07:29:15.936847+0000 mgr.y (mgr.14152) 334 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:17.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:16 smithi089 bash[21353]: cluster 2024-04-13T07:29:15.936847+0000 mgr.y (mgr.14152) 334 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:17.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:16 smithi089 bash[31522]: cluster 2024-04-13T07:29:15.936847+0000 mgr.y (mgr.14152) 334 : cluster 0 pgmap v317: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:17.513 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:29:19.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:18 smithi203 bash[23480]: cluster 2024-04-13T07:29:17.937490+0000 mgr.y (mgr.14152) 335 : cluster 0 pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:19.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:18 smithi089 bash[21353]: cluster 2024-04-13T07:29:17.937490+0000 mgr.y (mgr.14152) 335 : cluster 0 pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:19.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:18 smithi089 bash[31522]: cluster 2024-04-13T07:29:17.937490+0000 mgr.y (mgr.14152) 335 : cluster 0 pgmap v318: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:21.235 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:20 smithi203 bash[23480]: cluster 2024-04-13T07:29:19.938058+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:21.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:20 smithi089 bash[21353]: cluster 2024-04-13T07:29:19.938058+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:21.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:20 smithi089 bash[31522]: cluster 2024-04-13T07:29:19.938058+0000 mgr.y (mgr.14152) 336 : cluster 0 pgmap v319: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:23.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:22 smithi203 bash[23480]: cluster 2024-04-13T07:29:21.938651+0000 mgr.y (mgr.14152) 337 : cluster 0 pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:23.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:22 smithi089 bash[21353]: cluster 2024-04-13T07:29:21.938651+0000 mgr.y (mgr.14152) 337 : cluster 0 pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:23.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:22 smithi089 bash[31522]: cluster 2024-04-13T07:29:21.938651+0000 mgr.y (mgr.14152) 337 : cluster 0 pgmap v320: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:24.758 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled node-exporter update... 2024-04-13T07:29:26.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:25 smithi203 bash[23480]: cluster 2024-04-13T07:29:23.939234+0000 mgr.y (mgr.14152) 338 : cluster 0 pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:26.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:25 smithi203 bash[23480]: audit 2024-04-13T07:29:24.751749+0000 mgr.y (mgr.14152) 339 : audit 0 from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi089=a;smithi203=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:26.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:25 smithi203 bash[23480]: cephadm 2024-04-13T07:29:24.755436+0000 mgr.y (mgr.14152) 340 : cephadm 1 Saving service node-exporter spec with placement smithi089=a;smithi203=b;count:2 2024-04-13T07:29:26.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:25 smithi203 bash[23480]: audit 2024-04-13T07:29:24.763041+0000 mon.a (mon.0) 630 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:26.035 DEBUG:teuthology.orchestra.run.smithi089:node-exporter.a> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@node-exporter.a.service 2024-04-13T07:29:26.038 DEBUG:teuthology.orchestra.run.smithi203:node-exporter.b> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@node-exporter.b.service 2024-04-13T07:29:26.040 INFO:tasks.cephadm:Adding alertmanager.a on smithi089 2024-04-13T07:29:26.040 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch apply alertmanager '1;smithi089=a' 2024-04-13T07:29:26.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:25 smithi089 bash[21353]: cluster 2024-04-13T07:29:23.939234+0000 mgr.y (mgr.14152) 338 : cluster 0 pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:26.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:25 smithi089 bash[21353]: audit 2024-04-13T07:29:24.751749+0000 mgr.y (mgr.14152) 339 : audit 0 from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi089=a;smithi203=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:26.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:25 smithi089 bash[21353]: cephadm 2024-04-13T07:29:24.755436+0000 mgr.y (mgr.14152) 340 : cephadm 1 Saving service node-exporter spec with placement smithi089=a;smithi203=b;count:2 2024-04-13T07:29:26.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:25 smithi089 bash[21353]: audit 2024-04-13T07:29:24.763041+0000 mon.a (mon.0) 630 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:26.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:25 smithi089 bash[31522]: cluster 2024-04-13T07:29:23.939234+0000 mgr.y (mgr.14152) 338 : cluster 0 pgmap v321: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:26.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:25 smithi089 bash[31522]: audit 2024-04-13T07:29:24.751749+0000 mgr.y (mgr.14152) 339 : audit 0 from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi089=a;smithi203=b", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:26.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:25 smithi089 bash[31522]: cephadm 2024-04-13T07:29:24.755436+0000 mgr.y (mgr.14152) 340 : cephadm 1 Saving service node-exporter spec with placement smithi089=a;smithi203=b;count:2 2024-04-13T07:29:26.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:25 smithi089 bash[31522]: audit 2024-04-13T07:29:24.763041+0000 mon.a (mon.0) 630 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:28.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:27 smithi203 bash[23480]: cluster 2024-04-13T07:29:25.939776+0000 mgr.y (mgr.14152) 341 : cluster 0 pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:28.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:27 smithi089 bash[21353]: cluster 2024-04-13T07:29:25.939776+0000 mgr.y (mgr.14152) 341 : cluster 0 pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:28.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:27 smithi089 bash[31522]: cluster 2024-04-13T07:29:25.939776+0000 mgr.y (mgr.14152) 341 : cluster 0 pgmap v322: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:29.332 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.333 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.333 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.333 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.333 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.334 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.596 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.596 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.597 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.597 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.597 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.598 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.598 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.598 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.598 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.598 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:29.599 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:29 smithi203 systemd[1]: Started Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:29:30.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:29 smithi203 bash[23480]: cluster 2024-04-13T07:29:27.940345+0000 mgr.y (mgr.14152) 342 : cluster 0 pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:30.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:29 smithi203 bash[23480]: audit 2024-04-13T07:29:29.645771+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:29 smithi203 bash[23480]: audit 2024-04-13T07:29:29.656955+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:29 smithi203 bash[23480]: audit 2024-04-13T07:29:29.716255+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:29 smithi203 bash[23480]: audit 2024-04-13T07:29:29.725466+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-13T07:29:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[31522]: cluster 2024-04-13T07:29:27.940345+0000 mgr.y (mgr.14152) 342 : cluster 0 pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[31522]: audit 2024-04-13T07:29:29.645771+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[31522]: audit 2024-04-13T07:29:29.656955+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[31522]: audit 2024-04-13T07:29:29.716255+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[31522]: audit 2024-04-13T07:29:29.725466+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-13T07:29:30.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[21353]: cluster 2024-04-13T07:29:27.940345+0000 mgr.y (mgr.14152) 342 : cluster 0 pgmap v323: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:30.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[21353]: audit 2024-04-13T07:29:29.645771+0000 mon.a (mon.0) 631 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[21353]: audit 2024-04-13T07:29:29.656955+0000 mon.a (mon.0) 632 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[21353]: audit 2024-04-13T07:29:29.716255+0000 mon.a (mon.0) 633 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:30.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:29 smithi089 bash[21353]: audit 2024-04-13T07:29:29.725466+0000 mon.a (mon.0) 634 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2024-04-13T07:29:30.761 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:29:31.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:30 smithi203 bash[23480]: audit 2024-04-13T07:29:29.946685+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:31.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:30 smithi203 bash[23480]: audit 2024-04-13T07:29:30.727760+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-13T07:29:31.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:30 smithi203 bash[23480]: cluster 2024-04-13T07:29:30.727865+0000 mon.a (mon.0) 637 : cluster 0 mgrmap e16: y(active, since 9m), standbys: x 2024-04-13T07:29:31.017 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:30 smithi203 bash[24428]: ignoring --setuser ceph since I am not root 2024-04-13T07:29:31.017 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:30 smithi203 bash[24428]: ignoring --setgroup ceph since I am not root 2024-04-13T07:29:31.017 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:30 smithi203 bash[24428]: debug 2024-04-13T07:29:30.776+0000 7f39268ce700 1 -- 172.21.15.203:0/3005664826 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c8170041a0 con 0x55c817d86800 2024-04-13T07:29:31.018 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:30 smithi203 bash[24428]: debug 2024-04-13T07:29:30.924+0000 7f3930634200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:29:31.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[21606]: ignoring --setuser ceph since I am not root 2024-04-13T07:29:31.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[21606]: ignoring --setgroup ceph since I am not root 2024-04-13T07:29:31.050 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[21606]: debug 2024-04-13T07:29:30.927+0000 7f012616b200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:29:31.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[31522]: audit 2024-04-13T07:29:29.946685+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:31.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[31522]: audit 2024-04-13T07:29:30.727760+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-13T07:29:31.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[31522]: cluster 2024-04-13T07:29:30.727865+0000 mon.a (mon.0) 637 : cluster 0 mgrmap e16: y(active, since 9m), standbys: x 2024-04-13T07:29:31.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[21353]: audit 2024-04-13T07:29:29.946685+0000 mon.a (mon.0) 635 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' 2024-04-13T07:29:31.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[21353]: audit 2024-04-13T07:29:30.727760+0000 mon.a (mon.0) 636 : audit 1 from='mgr.14152 172.21.15.89:0/1509791138' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2024-04-13T07:29:31.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:30 smithi089 bash[21353]: cluster 2024-04-13T07:29:30.727865+0000 mon.a (mon.0) 637 : cluster 0 mgrmap e16: y(active, since 9m), standbys: x 2024-04-13T07:29:31.517 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.026Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.026Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.026Z caller=main.go:570 level=info host_details="(Linux 5.15.0-102-generic #112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024 x86_64 smithi203 (none))" 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.026Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.026Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.028Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.028Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.029Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.029Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.033Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.033Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.842µs 2024-04-13T07:29:31.518 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.033Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.033Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.034Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.771µs wal_replay_duration=232.575µs wbl_replay_duration=97ns total_replay_duration=272.925µs 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.035Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.035Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.035Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.036Z caller=refresh.go:80 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.036Z caller=refresh.go:80 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.036Z caller=refresh.go:80 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.055Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=20.00948ms db_storage=818ns remote_storage=1.285µs web_handler=369ns query_engine=654ns scrape=295.174µs scrape_sd=98.533µs notify=815ns notify_sd=1.437µs rules=19.301052ms tracing=7.814µs 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.055Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-13T07:29:31.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:31 smithi203 bash[48001]: ts=2024-04-13T07:29:31.055Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-13T07:29:32.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:32 smithi089 bash[21606]: debug 2024-04-13T07:29:32.491+0000 7f012616b200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:29:32.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:32 smithi089 bash[21606]: debug 2024-04-13T07:29:32.587+0000 7f012616b200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:29:32.864 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:32 smithi203 bash[24428]: debug 2024-04-13T07:29:32.516+0000 7f3930634200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:29:32.864 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:32 smithi203 bash[24428]: debug 2024-04-13T07:29:32.616+0000 7f3930634200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:29:33.195 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:32 smithi203 bash[24428]: debug 2024-04-13T07:29:32.864+0000 7f3930634200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:29:33.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:32 smithi089 bash[21606]: debug 2024-04-13T07:29:32.835+0000 7f012616b200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:29:34.017 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:33 smithi203 bash[24428]: debug 2024-04-13T07:29:33.660+0000 7f3930634200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:29:34.017 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:33 smithi203 bash[24428]: debug 2024-04-13T07:29:33.768+0000 7f3930634200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:29:34.017 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:33 smithi203 bash[24428]: debug 2024-04-13T07:29:33.876+0000 7f3930634200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:29:34.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:33 smithi089 bash[21606]: debug 2024-04-13T07:29:33.623+0000 7f012616b200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:29:34.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:33 smithi089 bash[21606]: debug 2024-04-13T07:29:33.735+0000 7f012616b200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:29:34.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:33 smithi089 bash[21606]: debug 2024-04-13T07:29:33.839+0000 7f012616b200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:29:34.455 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:34 smithi203 bash[24428]: debug 2024-04-13T07:29:34.232+0000 7f3930634200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:29:34.456 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:34 smithi203 bash[24428]: debug 2024-04-13T07:29:34.348+0000 7f3930634200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:29:34.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:34 smithi089 bash[21606]: debug 2024-04-13T07:29:34.195+0000 7f012616b200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:29:34.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:34 smithi089 bash[21606]: debug 2024-04-13T07:29:34.311+0000 7f012616b200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:29:34.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:34 smithi089 bash[21606]: debug 2024-04-13T07:29:34.415+0000 7f012616b200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:29:34.767 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:34 smithi203 bash[24428]: debug 2024-04-13T07:29:34.456+0000 7f3930634200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:29:35.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:34 smithi089 bash[21606]: debug 2024-04-13T07:29:34.775+0000 7f012616b200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:29:35.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:35 smithi089 bash[21606]: debug 2024-04-13T07:29:35.015+0000 7f012616b200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:29:35.267 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:34 smithi203 bash[24428]: debug 2024-04-13T07:29:34.808+0000 7f3930634200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:29:35.267 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:35 smithi203 bash[24428]: debug 2024-04-13T07:29:35.048+0000 7f3930634200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:29:35.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:35 smithi089 bash[21606]: debug 2024-04-13T07:29:35.279+0000 7f012616b200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:29:35.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:35 smithi089 bash[21606]: debug 2024-04-13T07:29:35.387+0000 7f012616b200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:29:35.690 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:35 smithi203 bash[24428]: debug 2024-04-13T07:29:35.308+0000 7f3930634200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:29:35.691 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:35 smithi203 bash[24428]: debug 2024-04-13T07:29:35.420+0000 7f3930634200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:29:35.691 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:35 smithi203 bash[24428]: debug 2024-04-13T07:29:35.688+0000 7f3930634200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:29:36.017 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:35 smithi203 bash[24428]: debug 2024-04-13T07:29:35.932+0000 7f3930634200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:29:36.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:35 smithi089 bash[21606]: debug 2024-04-13T07:29:35.667+0000 7f012616b200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:29:36.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:35 smithi089 bash[21606]: debug 2024-04-13T07:29:35.907+0000 7f012616b200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:29:36.688 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:36 smithi089 bash[21606]: debug 2024-04-13T07:29:36.319+0000 7f012616b200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:29:36.688 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:36 smithi089 bash[21606]: debug 2024-04-13T07:29:36.439+0000 7f012616b200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:29:36.688 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:36 smithi089 bash[21606]: debug 2024-04-13T07:29:36.687+0000 7f012616b200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:29:36.699 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:36 smithi203 bash[24428]: debug 2024-04-13T07:29:36.340+0000 7f3930634200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:29:36.699 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:36 smithi203 bash[24428]: debug 2024-04-13T07:29:36.456+0000 7f3930634200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:29:37.016 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:36 smithi203 bash[24428]: debug 2024-04-13T07:29:36.700+0000 7f3930634200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:29:38.016 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:37 smithi203 bash[24428]: debug 2024-04-13T07:29:37.700+0000 7f3930634200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:29:38.017 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:37 smithi203 bash[24428]: debug 2024-04-13T07:29:37.812+0000 7f3930634200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:29:38.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:37 smithi089 bash[21606]: debug 2024-04-13T07:29:37.699+0000 7f012616b200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:29:38.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:37 smithi089 bash[21606]: debug 2024-04-13T07:29:37.815+0000 7f012616b200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:29:38.517 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: debug 2024-04-13T07:29:38.100+0000 7f3930634200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:29:38.517 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: debug 2024-04-13T07:29:38.216+0000 7f3930634200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:29:38.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21606]: debug 2024-04-13T07:29:38.111+0000 7f012616b200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:29:38.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21606]: debug 2024-04-13T07:29:38.227+0000 7f012616b200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:29:38.939 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21606]: debug 2024-04-13T07:29:38.679+0000 7f012616b200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:29:38.939 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21606]: debug 2024-04-13T07:29:38.799+0000 7f012616b200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:29:38.940 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21606]: [13/Apr/2024:07:29:38] ENGINE Bus STARTING 2024-04-13T07:29:38.940 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: cluster 2024-04-13T07:29:38.802864+0000 mon.a (mon.0) 638 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:29:38.940 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: cluster 2024-04-13T07:29:38.803047+0000 mon.a (mon.0) 639 : cluster 0 Standby manager daemon x started 2024-04-13T07:29:38.940 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.807886+0000 mon.b (mon.1) 25 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.809328+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.812729+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.813530+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: cluster 2024-04-13T07:29:38.824617+0000 mon.a (mon.0) 640 : cluster 1 Active manager daemon y restarted 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: cluster 2024-04-13T07:29:38.825142+0000 mon.a (mon.0) 641 : cluster 1 Activating manager daemon y 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: cluster 2024-04-13T07:29:38.839167+0000 mon.a (mon.0) 642 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: cluster 2024-04-13T07:29:38.839724+0000 mon.a (mon.0) 643 : cluster 0 mgrmap e17: y(active, starting, since 0.0147782s), standbys: x 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.841183+0000 mon.a (mon.0) 644 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.841360+0000 mon.a (mon.0) 645 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.841508+0000 mon.a (mon.0) 646 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.843919+0000 mon.a (mon.0) 647 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:29:38.941 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.844163+0000 mon.a (mon.0) 648 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.844627+0000 mon.a (mon.0) 649 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.845018+0000 mon.a (mon.0) 650 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.845278+0000 mon.a (mon.0) 651 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.845525+0000 mon.a (mon.0) 652 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.845772+0000 mon.a (mon.0) 653 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.846201+0000 mon.a (mon.0) 654 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.846487+0000 mon.a (mon.0) 655 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.846712+0000 mon.a (mon.0) 656 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.847039+0000 mon.a (mon.0) 657 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.847199+0000 mon.a (mon.0) 658 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:29:38.942 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[21353]: audit 2024-04-13T07:29:38.847730+0000 mon.a (mon.0) 659 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:29:38.943 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: cluster 2024-04-13T07:29:38.802864+0000 mon.a (mon.0) 638 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:29:38.943 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: cluster 2024-04-13T07:29:38.803047+0000 mon.a (mon.0) 639 : cluster 0 Standby manager daemon x started 2024-04-13T07:29:38.943 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.807886+0000 mon.b (mon.1) 25 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:29:38.943 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.809328+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:29:38.943 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.812729+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:29:38.943 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.813530+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:29:38.943 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: cluster 2024-04-13T07:29:38.824617+0000 mon.a (mon.0) 640 : cluster 1 Active manager daemon y restarted 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: cluster 2024-04-13T07:29:38.825142+0000 mon.a (mon.0) 641 : cluster 1 Activating manager daemon y 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: cluster 2024-04-13T07:29:38.839167+0000 mon.a (mon.0) 642 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: cluster 2024-04-13T07:29:38.839724+0000 mon.a (mon.0) 643 : cluster 0 mgrmap e17: y(active, starting, since 0.0147782s), standbys: x 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.841183+0000 mon.a (mon.0) 644 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.841360+0000 mon.a (mon.0) 645 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.841508+0000 mon.a (mon.0) 646 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.843919+0000 mon.a (mon.0) 647 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.844163+0000 mon.a (mon.0) 648 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.844627+0000 mon.a (mon.0) 649 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.845018+0000 mon.a (mon.0) 650 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.845278+0000 mon.a (mon.0) 651 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:29:38.944 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.845525+0000 mon.a (mon.0) 652 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:29:38.945 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.845772+0000 mon.a (mon.0) 653 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:29:38.945 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.846201+0000 mon.a (mon.0) 654 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:29:38.945 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.846487+0000 mon.a (mon.0) 655 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:29:38.945 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.846712+0000 mon.a (mon.0) 656 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:29:38.945 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.847039+0000 mon.a (mon.0) 657 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:29:38.945 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.847199+0000 mon.a (mon.0) 658 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:29:38.945 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:38 smithi089 bash[31522]: audit 2024-04-13T07:29:38.847730+0000 mon.a (mon.0) 659 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:29:38.945 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: debug 2024-04-13T07:29:38.660+0000 7f3930634200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:29:38.946 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: debug 2024-04-13T07:29:38.776+0000 7f3930634200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:29:38.946 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: [13/Apr/2024:07:29:38] ENGINE Bus STARTING 2024-04-13T07:29:38.946 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: CherryPy Checker: 2024-04-13T07:29:38.946 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: The Application mounted at '' has an empty config. 2024-04-13T07:29:38.946 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: [13/Apr/2024:07:29:38] ENGINE Serving on http://:::9283 2024-04-13T07:29:38.946 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[24428]: [13/Apr/2024:07:29:38] ENGINE Bus STARTED 2024-04-13T07:29:38.946 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: cluster 2024-04-13T07:29:38.802864+0000 mon.a (mon.0) 638 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: cluster 2024-04-13T07:29:38.803047+0000 mon.a (mon.0) 639 : cluster 0 Standby manager daemon x started 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.807886+0000 mon.b (mon.1) 25 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.809328+0000 mon.b (mon.1) 26 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.812729+0000 mon.b (mon.1) 27 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.813530+0000 mon.b (mon.1) 28 : audit 0 from='mgr.? 172.21.15.203:0/881912471' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: cluster 2024-04-13T07:29:38.824617+0000 mon.a (mon.0) 640 : cluster 1 Active manager daemon y restarted 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: cluster 2024-04-13T07:29:38.825142+0000 mon.a (mon.0) 641 : cluster 1 Activating manager daemon y 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: cluster 2024-04-13T07:29:38.839167+0000 mon.a (mon.0) 642 : cluster 0 osdmap e60: 8 total, 8 up, 8 in 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: cluster 2024-04-13T07:29:38.839724+0000 mon.a (mon.0) 643 : cluster 0 mgrmap e17: y(active, starting, since 0.0147782s), standbys: x 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.841183+0000 mon.a (mon.0) 644 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.841360+0000 mon.a (mon.0) 645 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:29:38.947 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.841508+0000 mon.a (mon.0) 646 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.843919+0000 mon.a (mon.0) 647 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.844163+0000 mon.a (mon.0) 648 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.844627+0000 mon.a (mon.0) 649 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.845018+0000 mon.a (mon.0) 650 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.845278+0000 mon.a (mon.0) 651 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.845525+0000 mon.a (mon.0) 652 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.845772+0000 mon.a (mon.0) 653 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.846201+0000 mon.a (mon.0) 654 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.846487+0000 mon.a (mon.0) 655 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.846712+0000 mon.a (mon.0) 656 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.847039+0000 mon.a (mon.0) 657 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.847199+0000 mon.a (mon.0) 658 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:29:38.948 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:38 smithi203 bash[23480]: audit 2024-04-13T07:29:38.847730+0000 mon.a (mon.0) 659 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:29:39.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21606]: CherryPy Checker: 2024-04-13T07:29:39.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21606]: The Application mounted at '' has an empty config. 2024-04-13T07:29:39.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21606]: [13/Apr/2024:07:29:39] ENGINE Serving on http://:::9283 2024-04-13T07:29:39.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21606]: [13/Apr/2024:07:29:39] ENGINE Bus STARTED 2024-04-13T07:29:39.885 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled alertmanager update... 2024-04-13T07:29:40.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:39 smithi203 bash[23480]: cluster 2024-04-13T07:29:38.873436+0000 mon.a (mon.0) 660 : cluster 1 Manager daemon y is now available 2024-04-13T07:29:40.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:39 smithi203 bash[23480]: audit 2024-04-13T07:29:38.896583+0000 mon.a (mon.0) 661 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:40.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:39 smithi203 bash[23480]: audit 2024-04-13T07:29:38.919666+0000 mon.a (mon.0) 662 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:40.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:39 smithi203 bash[23480]: audit 2024-04-13T07:29:38.923850+0000 mon.a (mon.0) 663 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:29:40.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:39 smithi203 bash[23480]: audit 2024-04-13T07:29:38.929246+0000 mon.a (mon.0) 664 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:29:40.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:39 smithi203 bash[23480]: audit 2024-04-13T07:29:38.940094+0000 mon.a (mon.0) 665 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:29:40.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:39 smithi203 bash[23480]: cephadm 2024-04-13T07:29:39.684637+0000 mgr.y (mgr.14373) 1 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Bus STARTING 2024-04-13T07:29:40.181 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:39 smithi203 bash[23480]: cephadm 2024-04-13T07:29:39.786534+0000 mgr.y (mgr.14373) 2 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21353]: cluster 2024-04-13T07:29:38.873436+0000 mon.a (mon.0) 660 : cluster 1 Manager daemon y is now available 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21353]: audit 2024-04-13T07:29:38.896583+0000 mon.a (mon.0) 661 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21353]: audit 2024-04-13T07:29:38.919666+0000 mon.a (mon.0) 662 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21353]: audit 2024-04-13T07:29:38.923850+0000 mon.a (mon.0) 663 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21353]: audit 2024-04-13T07:29:38.929246+0000 mon.a (mon.0) 664 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21353]: audit 2024-04-13T07:29:38.940094+0000 mon.a (mon.0) 665 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21353]: cephadm 2024-04-13T07:29:39.684637+0000 mgr.y (mgr.14373) 1 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Bus STARTING 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[21353]: cephadm 2024-04-13T07:29:39.786534+0000 mgr.y (mgr.14373) 2 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[31522]: cluster 2024-04-13T07:29:38.873436+0000 mon.a (mon.0) 660 : cluster 1 Manager daemon y is now available 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[31522]: audit 2024-04-13T07:29:38.896583+0000 mon.a (mon.0) 661 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[31522]: audit 2024-04-13T07:29:38.919666+0000 mon.a (mon.0) 662 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:40.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[31522]: audit 2024-04-13T07:29:38.923850+0000 mon.a (mon.0) 663 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:29:40.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[31522]: audit 2024-04-13T07:29:38.929246+0000 mon.a (mon.0) 664 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:29:40.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[31522]: audit 2024-04-13T07:29:38.940094+0000 mon.a (mon.0) 665 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:29:40.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[31522]: cephadm 2024-04-13T07:29:39.684637+0000 mgr.y (mgr.14373) 1 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Bus STARTING 2024-04-13T07:29:40.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:39 smithi089 bash[31522]: cephadm 2024-04-13T07:29:39.786534+0000 mgr.y (mgr.14373) 2 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:29:40.840 DEBUG:teuthology.orchestra.run.smithi089:alertmanager.a> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@alertmanager.a.service 2024-04-13T07:29:40.843 INFO:tasks.cephadm:Adding grafana.a on smithi203 2024-04-13T07:29:40.843 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph orch apply grafana '1;smithi203=a' 2024-04-13T07:29:41.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:40 smithi203 bash[23480]: cluster 2024-04-13T07:29:39.868746+0000 mon.a (mon.0) 666 : cluster 0 mgrmap e18: y(active, since 1.0438s), standbys: x 2024-04-13T07:29:41.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:40 smithi203 bash[23480]: audit 2024-04-13T07:29:39.881712+0000 mgr.y (mgr.14373) 3 : audit 0 from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi089=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:41.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:40 smithi203 bash[23480]: cluster 2024-04-13T07:29:39.882415+0000 mgr.y (mgr.14373) 4 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:41.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:40 smithi203 bash[23480]: cephadm 2024-04-13T07:29:39.885161+0000 mgr.y (mgr.14373) 5 : cephadm 1 Saving service alertmanager spec with placement smithi089=a;count:1 2024-04-13T07:29:41.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:40 smithi203 bash[23480]: audit 2024-04-13T07:29:39.889785+0000 mon.a (mon.0) 667 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:41.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:40 smithi203 bash[23480]: cephadm 2024-04-13T07:29:39.904762+0000 mgr.y (mgr.14373) 6 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:29:41.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:40 smithi203 bash[23480]: cephadm 2024-04-13T07:29:39.905009+0000 mgr.y (mgr.14373) 7 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Bus STARTED 2024-04-13T07:29:41.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[21353]: cluster 2024-04-13T07:29:39.868746+0000 mon.a (mon.0) 666 : cluster 0 mgrmap e18: y(active, since 1.0438s), standbys: x 2024-04-13T07:29:41.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[21353]: audit 2024-04-13T07:29:39.881712+0000 mgr.y (mgr.14373) 3 : audit 0 from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi089=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:41.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[21353]: cluster 2024-04-13T07:29:39.882415+0000 mgr.y (mgr.14373) 4 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:41.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[21353]: cephadm 2024-04-13T07:29:39.885161+0000 mgr.y (mgr.14373) 5 : cephadm 1 Saving service alertmanager spec with placement smithi089=a;count:1 2024-04-13T07:29:41.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[21353]: audit 2024-04-13T07:29:39.889785+0000 mon.a (mon.0) 667 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:41.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[21353]: cephadm 2024-04-13T07:29:39.904762+0000 mgr.y (mgr.14373) 6 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:29:41.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[21353]: cephadm 2024-04-13T07:29:39.905009+0000 mgr.y (mgr.14373) 7 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Bus STARTED 2024-04-13T07:29:41.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[31522]: cluster 2024-04-13T07:29:39.868746+0000 mon.a (mon.0) 666 : cluster 0 mgrmap e18: y(active, since 1.0438s), standbys: x 2024-04-13T07:29:41.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[31522]: audit 2024-04-13T07:29:39.881712+0000 mgr.y (mgr.14373) 3 : audit 0 from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi089=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:41.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[31522]: cluster 2024-04-13T07:29:39.882415+0000 mgr.y (mgr.14373) 4 : cluster 0 pgmap v3: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:41.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[31522]: cephadm 2024-04-13T07:29:39.885161+0000 mgr.y (mgr.14373) 5 : cephadm 1 Saving service alertmanager spec with placement smithi089=a;count:1 2024-04-13T07:29:41.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[31522]: audit 2024-04-13T07:29:39.889785+0000 mon.a (mon.0) 667 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:41.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[31522]: cephadm 2024-04-13T07:29:39.904762+0000 mgr.y (mgr.14373) 6 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:29:41.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:40 smithi089 bash[31522]: cephadm 2024-04-13T07:29:39.905009+0000 mgr.y (mgr.14373) 7 : cephadm 1 [13/Apr/2024:07:29:39] ENGINE Bus STARTED 2024-04-13T07:29:42.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:41 smithi203 bash[23480]: cluster 2024-04-13T07:29:40.846108+0000 mgr.y (mgr.14373) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:42.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:41 smithi203 bash[23480]: cluster 2024-04-13T07:29:40.899789+0000 mon.a (mon.0) 668 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-04-13T07:29:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:41 smithi089 bash[21353]: cluster 2024-04-13T07:29:40.846108+0000 mgr.y (mgr.14373) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:41 smithi089 bash[21353]: cluster 2024-04-13T07:29:40.899789+0000 mon.a (mon.0) 668 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-04-13T07:29:42.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:41 smithi089 bash[31522]: cluster 2024-04-13T07:29:40.846108+0000 mgr.y (mgr.14373) 8 : cluster 0 pgmap v4: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:42.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:41 smithi089 bash[31522]: cluster 2024-04-13T07:29:40.899789+0000 mon.a (mon.0) 668 : cluster 0 mgrmap e19: y(active, since 2s), standbys: x 2024-04-13T07:29:44.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:43 smithi203 bash[23480]: cluster 2024-04-13T07:29:42.846683+0000 mgr.y (mgr.14373) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:44.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:43 smithi203 bash[23480]: cluster 2024-04-13T07:29:42.912492+0000 mon.a (mon.0) 669 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-04-13T07:29:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:43 smithi089 bash[31522]: cluster 2024-04-13T07:29:42.846683+0000 mgr.y (mgr.14373) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:43 smithi089 bash[31522]: cluster 2024-04-13T07:29:42.912492+0000 mon.a (mon.0) 669 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-04-13T07:29:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:43 smithi089 bash[21353]: cluster 2024-04-13T07:29:42.846683+0000 mgr.y (mgr.14373) 9 : cluster 0 pgmap v5: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:43 smithi089 bash[21353]: cluster 2024-04-13T07:29:42.912492+0000 mon.a (mon.0) 669 : cluster 0 mgrmap e20: y(active, since 4s), standbys: x 2024-04-13T07:29:45.377 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:29:45.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:45 smithi203 bash[23480]: audit 2024-04-13T07:29:44.502921+0000 mon.a (mon.0) 670 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:45 smithi203 bash[23480]: audit 2024-04-13T07:29:44.508506+0000 mon.a (mon.0) 671 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:45 smithi203 bash[23480]: audit 2024-04-13T07:29:44.855243+0000 mon.a (mon.0) 672 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:45 smithi203 bash[23480]: audit 2024-04-13T07:29:44.861071+0000 mon.a (mon.0) 673 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:45 smithi203 bash[23480]: audit 2024-04-13T07:29:45.356041+0000 mon.a (mon.0) 674 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:45 smithi203 bash[23480]: audit 2024-04-13T07:29:45.361650+0000 mon.a (mon.0) 675 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[21353]: audit 2024-04-13T07:29:44.502921+0000 mon.a (mon.0) 670 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[21353]: audit 2024-04-13T07:29:44.508506+0000 mon.a (mon.0) 671 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[21353]: audit 2024-04-13T07:29:44.855243+0000 mon.a (mon.0) 672 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[21353]: audit 2024-04-13T07:29:44.861071+0000 mon.a (mon.0) 673 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[21353]: audit 2024-04-13T07:29:45.356041+0000 mon.a (mon.0) 674 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[21353]: audit 2024-04-13T07:29:45.361650+0000 mon.a (mon.0) 675 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[31522]: audit 2024-04-13T07:29:44.502921+0000 mon.a (mon.0) 670 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[31522]: audit 2024-04-13T07:29:44.508506+0000 mon.a (mon.0) 671 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[31522]: audit 2024-04-13T07:29:44.855243+0000 mon.a (mon.0) 672 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[31522]: audit 2024-04-13T07:29:44.861071+0000 mon.a (mon.0) 673 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[31522]: audit 2024-04-13T07:29:45.356041+0000 mon.a (mon.0) 674 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:45.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:45 smithi089 bash[31522]: audit 2024-04-13T07:29:45.361650+0000 mon.a (mon.0) 675 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: cluster 2024-04-13T07:29:44.847377+0000 mgr.y (mgr.14373) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.818729+0000 mon.a (mon.0) 676 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.831413+0000 mon.a (mon.0) 677 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.834512+0000 mon.a (mon.0) 678 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.836243+0000 mon.a (mon.0) 679 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.837763+0000 mon.a (mon.0) 680 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.839316+0000 mon.a (mon.0) 681 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.851231+0000 mon.a (mon.0) 682 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.853372+0000 mon.a (mon.0) 683 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:29:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:46 smithi203 bash[23480]: audit 2024-04-13T07:29:45.855139+0000 mon.a (mon.0) 684 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:29:46.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: cluster 2024-04-13T07:29:44.847377+0000 mgr.y (mgr.14373) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:46.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.818729+0000 mon.a (mon.0) 676 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.831413+0000 mon.a (mon.0) 677 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.834512+0000 mon.a (mon.0) 678 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.836243+0000 mon.a (mon.0) 679 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.837763+0000 mon.a (mon.0) 680 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.839316+0000 mon.a (mon.0) 681 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.851231+0000 mon.a (mon.0) 682 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.853372+0000 mon.a (mon.0) 683 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[21353]: audit 2024-04-13T07:29:45.855139+0000 mon.a (mon.0) 684 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: cluster 2024-04-13T07:29:44.847377+0000 mgr.y (mgr.14373) 10 : cluster 0 pgmap v6: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.818729+0000 mon.a (mon.0) 676 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.831413+0000 mon.a (mon.0) 677 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.834512+0000 mon.a (mon.0) 678 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.836243+0000 mon.a (mon.0) 679 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.837763+0000 mon.a (mon.0) 680 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.839316+0000 mon.a (mon.0) 681 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:29:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.851231+0000 mon.a (mon.0) 682 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.853372+0000 mon.a (mon.0) 683 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:29:46.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:46 smithi089 bash[31522]: audit 2024-04-13T07:29:45.855139+0000 mon.a (mon.0) 684 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:29:47.549 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.549 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.550 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.550 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.551 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.551 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.551 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.551 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:47.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:45.840526+0000 mgr.y (mgr.14373) 11 : cephadm 1 Adjusting osd_memory_target on smithi203 to 4060M 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:45.857749+0000 mgr.y (mgr.14373) 12 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:45.858233+0000 mgr.y (mgr.14373) 13 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:46.123906+0000 mgr.y (mgr.14373) 14 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:46.124993+0000 mgr.y (mgr.14373) 15 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:46.288257+0000 mgr.y (mgr.14373) 16 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:46.290645+0000 mgr.y (mgr.14373) 17 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:46.453273+0000 mgr.y (mgr.14373) 18 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:46.455923+0000 mgr.y (mgr.14373) 19 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: audit 2024-04-13T07:29:46.626830+0000 mon.a (mon.0) 685 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: audit 2024-04-13T07:29:46.631943+0000 mon.a (mon.0) 686 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: audit 2024-04-13T07:29:46.637609+0000 mon.a (mon.0) 687 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: audit 2024-04-13T07:29:46.643381+0000 mon.a (mon.0) 688 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: audit 2024-04-13T07:29:46.650145+0000 mon.a (mon.0) 689 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:47.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:47 smithi203 bash[23480]: cephadm 2024-04-13T07:29:46.655043+0000 mgr.y (mgr.14373) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi089 2024-04-13T07:29:48.050 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:47 smithi089 systemd[1]: Started Ceph node-exporter.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:29:48.050 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[56011]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-04-13T07:29:48.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:45.840526+0000 mgr.y (mgr.14373) 11 : cephadm 1 Adjusting osd_memory_target on smithi203 to 4060M 2024-04-13T07:29:48.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:45.857749+0000 mgr.y (mgr.14373) 12 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:29:48.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:45.858233+0000 mgr.y (mgr.14373) 13 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:29:48.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:46.123906+0000 mgr.y (mgr.14373) 14 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:29:48.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:46.124993+0000 mgr.y (mgr.14373) 15 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:29:48.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:46.288257+0000 mgr.y (mgr.14373) 16 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:46.290645+0000 mgr.y (mgr.14373) 17 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:46.453273+0000 mgr.y (mgr.14373) 18 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:46.455923+0000 mgr.y (mgr.14373) 19 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: audit 2024-04-13T07:29:46.626830+0000 mon.a (mon.0) 685 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: audit 2024-04-13T07:29:46.631943+0000 mon.a (mon.0) 686 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: audit 2024-04-13T07:29:46.637609+0000 mon.a (mon.0) 687 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: audit 2024-04-13T07:29:46.643381+0000 mon.a (mon.0) 688 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: audit 2024-04-13T07:29:46.650145+0000 mon.a (mon.0) 689 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[21353]: cephadm 2024-04-13T07:29:46.655043+0000 mgr.y (mgr.14373) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi089 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:45.840526+0000 mgr.y (mgr.14373) 11 : cephadm 1 Adjusting osd_memory_target on smithi203 to 4060M 2024-04-13T07:29:48.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:45.857749+0000 mgr.y (mgr.14373) 12 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:45.858233+0000 mgr.y (mgr.14373) 13 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:46.123906+0000 mgr.y (mgr.14373) 14 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:46.124993+0000 mgr.y (mgr.14373) 15 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:46.288257+0000 mgr.y (mgr.14373) 16 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:46.290645+0000 mgr.y (mgr.14373) 17 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:46.453273+0000 mgr.y (mgr.14373) 18 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:46.455923+0000 mgr.y (mgr.14373) 19 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: audit 2024-04-13T07:29:46.626830+0000 mon.a (mon.0) 685 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: audit 2024-04-13T07:29:46.631943+0000 mon.a (mon.0) 686 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: audit 2024-04-13T07:29:46.637609+0000 mon.a (mon.0) 687 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: audit 2024-04-13T07:29:46.643381+0000 mon.a (mon.0) 688 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: audit 2024-04-13T07:29:46.650145+0000 mon.a (mon.0) 689 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:47 smithi089 bash[31522]: cephadm 2024-04-13T07:29:46.655043+0000 mgr.y (mgr.14373) 20 : cephadm 1 Deploying daemon node-exporter.a on smithi089 2024-04-13T07:29:48.206 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:47 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.468 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.468 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.468 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.469 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.469 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.469 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.469 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.549 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: v1.5.0: Pulling from prometheus/node-exporter 2024-04-13T07:29:48.549 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: 22b70bddd3ac: Pulling fs layer 2024-04-13T07:29:48.549 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: 5c12815fee55: Pulling fs layer 2024-04-13T07:29:48.550 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: c0e87333d380: Pulling fs layer 2024-04-13T07:29:48.721 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.721 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.722 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.722 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.722 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.722 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.723 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.723 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.723 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: Started Ceph node-exporter.b for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:29:48.723 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.723 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.723 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:48 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:48.723 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:48 smithi203 bash[23480]: cluster 2024-04-13T07:29:46.848005+0000 mgr.y (mgr.14373) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:29:48.724 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:48 smithi203 bash[23480]: audit 2024-04-13T07:29:47.656055+0000 mon.a (mon.0) 690 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.724 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:48 smithi203 bash[23480]: audit 2024-04-13T07:29:47.661610+0000 mon.a (mon.0) 691 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.724 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:48 smithi203 bash[23480]: audit 2024-04-13T07:29:47.676740+0000 mon.a (mon.0) 692 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:48.724 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:48 smithi203 bash[23480]: cephadm 2024-04-13T07:29:47.677479+0000 mgr.y (mgr.14373) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi203 2024-04-13T07:29:49.016 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:48 smithi203 bash[48925]: Unable to find image 'quay.io/prometheus/node-exporter:v1.5.0' locally 2024-04-13T07:29:49.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[31522]: cluster 2024-04-13T07:29:46.848005+0000 mgr.y (mgr.14373) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:29:49.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[31522]: audit 2024-04-13T07:29:47.656055+0000 mon.a (mon.0) 690 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[31522]: audit 2024-04-13T07:29:47.661610+0000 mon.a (mon.0) 691 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[31522]: audit 2024-04-13T07:29:47.676740+0000 mon.a (mon.0) 692 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[31522]: cephadm 2024-04-13T07:29:47.677479+0000 mgr.y (mgr.14373) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi203 2024-04-13T07:29:49.051 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: 5c12815fee55: Verifying Checksum 2024-04-13T07:29:49.051 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: 5c12815fee55: Download complete 2024-04-13T07:29:49.051 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: 22b70bddd3ac: Verifying Checksum 2024-04-13T07:29:49.051 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: 22b70bddd3ac: Download complete 2024-04-13T07:29:49.051 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: c0e87333d380: Verifying Checksum 2024-04-13T07:29:49.051 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: c0e87333d380: Download complete 2024-04-13T07:29:49.052 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[56011]: 22b70bddd3ac: Pull complete 2024-04-13T07:29:49.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[21353]: cluster 2024-04-13T07:29:46.848005+0000 mgr.y (mgr.14373) 21 : cluster 0 pgmap v7: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:29:49.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[21353]: audit 2024-04-13T07:29:47.656055+0000 mon.a (mon.0) 690 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[21353]: audit 2024-04-13T07:29:47.661610+0000 mon.a (mon.0) 691 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[21353]: audit 2024-04-13T07:29:47.676740+0000 mon.a (mon.0) 692 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:48 smithi089 bash[21353]: cephadm 2024-04-13T07:29:47.677479+0000 mgr.y (mgr.14373) 22 : cephadm 1 Deploying daemon node-exporter.b on smithi203 2024-04-13T07:29:49.092 INFO:teuthology.orchestra.run.smithi203.stdout:Scheduled grafana update... 2024-04-13T07:29:49.549 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[56011]: 5c12815fee55: Pull complete 2024-04-13T07:29:49.549 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:49 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:49.766 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:49 smithi203 bash[48925]: v1.5.0: Pulling from prometheus/node-exporter 2024-04-13T07:29:49.894 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[56011]: c0e87333d380: Pull complete 2024-04-13T07:29:49.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[31522]: audit 2024-04-13T07:29:48.763241+0000 mon.a (mon.0) 693 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[31522]: audit 2024-04-13T07:29:48.770272+0000 mon.a (mon.0) 694 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[31522]: audit 2024-04-13T07:29:48.798747+0000 mon.a (mon.0) 695 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[31522]: audit 2024-04-13T07:29:48.804827+0000 mon.a (mon.0) 696 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[31522]: audit 2024-04-13T07:29:48.815911+0000 mon.a (mon.0) 697 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[31522]: audit 2024-04-13T07:29:48.921519+0000 mon.a (mon.0) 698 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[31522]: audit 2024-04-13T07:29:49.096764+0000 mon.a (mon.0) 699 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.895 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[21353]: audit 2024-04-13T07:29:48.763241+0000 mon.a (mon.0) 693 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[21353]: audit 2024-04-13T07:29:48.770272+0000 mon.a (mon.0) 694 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[21353]: audit 2024-04-13T07:29:48.798747+0000 mon.a (mon.0) 695 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[21353]: audit 2024-04-13T07:29:48.804827+0000 mon.a (mon.0) 696 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[21353]: audit 2024-04-13T07:29:48.815911+0000 mon.a (mon.0) 697 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[21353]: audit 2024-04-13T07:29:48.921519+0000 mon.a (mon.0) 698 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:49.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[21353]: audit 2024-04-13T07:29:49.096764+0000 mon.a (mon.0) 699 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:50.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:49 smithi203 bash[23480]: audit 2024-04-13T07:29:48.763241+0000 mon.a (mon.0) 693 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:50.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:49 smithi203 bash[23480]: audit 2024-04-13T07:29:48.770272+0000 mon.a (mon.0) 694 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:50.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:49 smithi203 bash[23480]: audit 2024-04-13T07:29:48.798747+0000 mon.a (mon.0) 695 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:50.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:49 smithi203 bash[23480]: audit 2024-04-13T07:29:48.804827+0000 mon.a (mon.0) 696 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:50.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:49 smithi203 bash[23480]: audit 2024-04-13T07:29:48.815911+0000 mon.a (mon.0) 697 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:50.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:49 smithi203 bash[23480]: audit 2024-04-13T07:29:48.921519+0000 mon.a (mon.0) 698 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:50.131 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:49 smithi203 bash[23480]: audit 2024-04-13T07:29:49.096764+0000 mon.a (mon.0) 699 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:50.299 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[56011]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-04-13T07:29:50.299 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:49 smithi089 bash[56011]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-04-13T07:29:50.304 DEBUG:teuthology.orchestra.run.smithi203:grafana.a> sudo journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@grafana.a.service 2024-04-13T07:29:50.306 INFO:tasks.cephadm:Setting up client nodes... 2024-04-13T07:29:50.307 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-13T07:29:50.388 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: 22b70bddd3ac: Pulling fs layer 2024-04-13T07:29:50.388 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: 5c12815fee55: Pulling fs layer 2024-04-13T07:29:50.388 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: c0e87333d380: Pulling fs layer 2024-04-13T07:29:50.388 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: 22b70bddd3ac: Verifying Checksum 2024-04-13T07:29:50.388 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: 22b70bddd3ac: Download complete 2024-04-13T07:29:50.388 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: c0e87333d380: Verifying Checksum 2024-04-13T07:29:50.388 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: c0e87333d380: Download complete 2024-04-13T07:29:50.760 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: 5c12815fee55: Download complete 2024-04-13T07:29:50.761 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[48925]: 22b70bddd3ac: Pull complete 2024-04-13T07:29:51.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[23480]: cephadm 2024-04-13T07:29:48.844513+0000 mgr.y (mgr.14373) 23 : cephadm 1 Deploying daemon alertmanager.a on smithi089 2024-04-13T07:29:51.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[23480]: cluster 2024-04-13T07:29:48.848674+0000 mgr.y (mgr.14373) 24 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:29:51.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[23480]: audit 2024-04-13T07:29:49.089910+0000 mgr.y (mgr.14373) 25 : audit 0 from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:51.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:50 smithi203 bash[23480]: cephadm 2024-04-13T07:29:49.092487+0000 mgr.y (mgr.14373) 26 : cephadm 1 Saving service grafana spec with placement smithi203=a;count:1 2024-04-13T07:29:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:50 smithi089 bash[31522]: cephadm 2024-04-13T07:29:48.844513+0000 mgr.y (mgr.14373) 23 : cephadm 1 Deploying daemon alertmanager.a on smithi089 2024-04-13T07:29:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:50 smithi089 bash[31522]: cluster 2024-04-13T07:29:48.848674+0000 mgr.y (mgr.14373) 24 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:29:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:50 smithi089 bash[31522]: audit 2024-04-13T07:29:49.089910+0000 mgr.y (mgr.14373) 25 : audit 0 from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:50 smithi089 bash[31522]: cephadm 2024-04-13T07:29:49.092487+0000 mgr.y (mgr.14373) 26 : cephadm 1 Saving service grafana spec with placement smithi203=a;count:1 2024-04-13T07:29:51.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:50 smithi089 bash[21353]: cephadm 2024-04-13T07:29:48.844513+0000 mgr.y (mgr.14373) 23 : cephadm 1 Deploying daemon alertmanager.a on smithi089 2024-04-13T07:29:51.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:50 smithi089 bash[21353]: cluster 2024-04-13T07:29:48.848674+0000 mgr.y (mgr.14373) 24 : cluster 0 pgmap v8: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:29:51.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:50 smithi089 bash[21353]: audit 2024-04-13T07:29:49.089910+0000 mgr.y (mgr.14373) 25 : audit 0 from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi203=a", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:29:51.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:50 smithi089 bash[21353]: cephadm 2024-04-13T07:29:49.092487+0000 mgr.y (mgr.14373) 26 : cephadm 1 Saving service grafana spec with placement smithi203=a;count:1 2024-04-13T07:29:51.266 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:51 smithi203 bash[48925]: 5c12815fee55: Pull complete 2024-04-13T07:29:51.725 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:51 smithi203 bash[48925]: c0e87333d380: Pull complete 2024-04-13T07:29:51.725 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:51 smithi203 bash[48925]: Digest: sha256:39c642b2b337e38c18e80266fb14383754178202f40103646337722a594d984c 2024-04-13T07:29:51.725 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:51 smithi203 bash[48925]: Status: Downloaded newer image for quay.io/prometheus/node-exporter:v1.5.0 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.389Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.389Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=arp 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-13T07:29:51.800 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=edac 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.390Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-13T07:29:51.801 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=os 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-13T07:29:51.802 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=stat 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=time 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=uname 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-13T07:29:51.803 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-13T07:29:51.804 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:51 smithi089 bash[56011]: ts=2024-04-13T07:29:51.391Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-13T07:29:53.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:52 smithi203 bash[23480]: cluster 2024-04-13T07:29:50.849133+0000 mgr.y (mgr.14373) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:29:53.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:52 smithi089 bash[21353]: cluster 2024-04-13T07:29:50.849133+0000 mgr.y (mgr.14373) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:29:53.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:52 smithi089 bash[31522]: cluster 2024-04-13T07:29:50.849133+0000 mgr.y (mgr.14373) 27 : cluster 0 pgmap v9: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:29:53.517 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.242Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.242Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.242Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.243Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.243Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.243Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=arp 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=bcache 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=bonding 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=btrfs 2024-04-13T07:29:53.518 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=conntrack 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=cpu 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=cpufreq 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=diskstats 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=dmi 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=edac 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=entropy 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=fibrechannel 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=filefd 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=filesystem 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=hwmon 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=infiniband 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=ipvs 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=loadavg 2024-04-13T07:29:53.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=mdadm 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=meminfo 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=netclass 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=netdev 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=netstat 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=nfs 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=nfsd 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=nvme 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=os 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2024-04-13T07:29:53.520 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=pressure 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=rapl 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=schedstat 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=selinux 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=sockstat 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=softnet 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=stat 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=tapestats 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=textfile 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=thermal_zone 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=time 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=udp_queues 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=uname 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=vmstat 2024-04-13T07:29:53.521 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=xfs 2024-04-13T07:29:53.522 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=node_exporter.go:117 level=info collector=zfs 2024-04-13T07:29:53.522 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2024-04-13T07:29:53.522 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:29:53 smithi203 bash[48925]: ts=2024-04-13T07:29:53.244Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2024-04-13T07:29:55.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:54 smithi089 bash[21353]: cluster 2024-04-13T07:29:52.849661+0000 mgr.y (mgr.14373) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:55.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:54 smithi089 bash[21353]: audit 2024-04-13T07:29:53.924965+0000 mon.a (mon.0) 700 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:29:55.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:54 smithi089 bash[31522]: cluster 2024-04-13T07:29:52.849661+0000 mgr.y (mgr.14373) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:55.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:54 smithi089 bash[31522]: audit 2024-04-13T07:29:53.924965+0000 mon.a (mon.0) 700 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:29:55.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:54 smithi203 bash[23480]: cluster 2024-04-13T07:29:52.849661+0000 mgr.y (mgr.14373) 28 : cluster 0 pgmap v10: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:55.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:54 smithi203 bash[23480]: audit 2024-04-13T07:29:53.924965+0000 mon.a (mon.0) 700 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:29:56.014 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:29:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:56 smithi089 bash[21353]: cluster 2024-04-13T07:29:54.850268+0000 mgr.y (mgr.14373) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:56 smithi089 bash[31522]: cluster 2024-04-13T07:29:54.850268+0000 mgr.y (mgr.14373) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:57.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:56 smithi203 bash[23480]: cluster 2024-04-13T07:29:54.850268+0000 mgr.y (mgr.14373) 29 : cluster 0 pgmap v11: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:57.410 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.410 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.410 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.411 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.411 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.411 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.412 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.412 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.680 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.680 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.680 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.681 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.681 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.681 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.682 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.682 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.682 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.682 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.682 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:57.682 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:29:58.049 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:57 smithi089 systemd[1]: Started Ceph alertmanager.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:29:59.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:58 smithi203 bash[23480]: cluster 2024-04-13T07:29:56.850928+0000 mgr.y (mgr.14373) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:59.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:58 smithi203 bash[23480]: audit 2024-04-13T07:29:57.724261+0000 mon.a (mon.0) 701 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:58 smithi203 bash[23480]: audit 2024-04-13T07:29:57.730040+0000 mon.a (mon.0) 702 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:58 smithi203 bash[23480]: audit 2024-04-13T07:29:57.785486+0000 mon.a (mon.0) 703 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:58 smithi203 bash[23480]: audit 2024-04-13T07:29:57.790456+0000 mon.a (mon.0) 704 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:58 smithi203 bash[23480]: audit 2024-04-13T07:29:57.839327+0000 mon.a (mon.0) 705 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:59.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[31522]: cluster 2024-04-13T07:29:56.850928+0000 mgr.y (mgr.14373) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:59.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[31522]: audit 2024-04-13T07:29:57.724261+0000 mon.a (mon.0) 701 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[31522]: audit 2024-04-13T07:29:57.730040+0000 mon.a (mon.0) 702 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[31522]: audit 2024-04-13T07:29:57.785486+0000 mon.a (mon.0) 703 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[31522]: audit 2024-04-13T07:29:57.790456+0000 mon.a (mon.0) 704 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[31522]: audit 2024-04-13T07:29:57.839327+0000 mon.a (mon.0) 705 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:59.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[21353]: cluster 2024-04-13T07:29:56.850928+0000 mgr.y (mgr.14373) 30 : cluster 0 pgmap v12: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:29:59.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[21353]: audit 2024-04-13T07:29:57.724261+0000 mon.a (mon.0) 701 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[21353]: audit 2024-04-13T07:29:57.730040+0000 mon.a (mon.0) 702 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[21353]: audit 2024-04-13T07:29:57.785486+0000 mon.a (mon.0) 703 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[21353]: audit 2024-04-13T07:29:57.790456+0000 mon.a (mon.0) 704 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:29:59.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:58 smithi089 bash[21353]: audit 2024-04-13T07:29:57.839327+0000 mon.a (mon.0) 705 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:29:59.795 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[56462]: ts=2024-04-13T07:29:59.564Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-13T07:29:59.795 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[56462]: ts=2024-04-13T07:29:59.564Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-13T07:29:59.795 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[56462]: ts=2024-04-13T07:29:59.566Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.89 port=9094 2024-04-13T07:29:59.795 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[56462]: ts=2024-04-13T07:29:59.567Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-13T07:29:59.795 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[56462]: ts=2024-04-13T07:29:59.597Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-13T07:29:59.795 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[56462]: ts=2024-04-13T07:29:59.598Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-13T07:29:59.796 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[56462]: ts=2024-04-13T07:29:59.601Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-13T07:29:59.796 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[56462]: ts=2024-04-13T07:29:59.601Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-13T07:30:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[21353]: cluster 2024-04-13T07:29:58.851357+0000 mgr.y (mgr.14373) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[21353]: audit 2024-04-13T07:29:58.936218+0000 mon.a (mon.0) 706 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:00.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[31522]: cluster 2024-04-13T07:29:58.851357+0000 mgr.y (mgr.14373) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:00.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:29:59 smithi089 bash[31522]: audit 2024-04-13T07:29:58.936218+0000 mon.a (mon.0) 706 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:00.263 INFO:teuthology.orchestra.run.smithi089.stdout:[client.0] 2024-04-13T07:30:00.263 INFO:teuthology.orchestra.run.smithi089.stdout: key = AQB4NBpm5PmfDxAAqWag8jwauSXcOxkiqy7W8Q== 2024-04-13T07:30:00.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:59 smithi203 bash[23480]: cluster 2024-04-13T07:29:58.851357+0000 mgr.y (mgr.14373) 31 : cluster 0 pgmap v13: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:00.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:29:59 smithi203 bash[23480]: audit 2024-04-13T07:29:58.936218+0000 mon.a (mon.0) 706 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:00.922 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:30:00.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2024-04-13T07:30:00.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2024-04-13T07:30:00.946 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2024-04-13T07:30:01.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:00 smithi203 bash[23480]: cluster 2024-04-13T07:30:00.000121+0000 mon.a (mon.0) 707 : cluster 1 overall HEALTH_OK 2024-04-13T07:30:01.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:00 smithi203 bash[23480]: audit 2024-04-13T07:30:00.261173+0000 mon.c (mon.2) 14 : audit 1 from='client.? 172.21.15.89:0/1525652961' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:01.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:00 smithi203 bash[23480]: audit 2024-04-13T07:30:00.261925+0000 mon.a (mon.0) 708 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:01.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:00 smithi203 bash[23480]: audit 2024-04-13T07:30:00.268540+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-13T07:30:01.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:00 smithi089 bash[21353]: cluster 2024-04-13T07:30:00.000121+0000 mon.a (mon.0) 707 : cluster 1 overall HEALTH_OK 2024-04-13T07:30:01.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:00 smithi089 bash[21353]: audit 2024-04-13T07:30:00.261173+0000 mon.c (mon.2) 14 : audit 1 from='client.? 172.21.15.89:0/1525652961' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:01.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:00 smithi089 bash[21353]: audit 2024-04-13T07:30:00.261925+0000 mon.a (mon.0) 708 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:01.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:00 smithi089 bash[21353]: audit 2024-04-13T07:30:00.268540+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-13T07:30:01.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:00 smithi089 bash[31522]: cluster 2024-04-13T07:30:00.000121+0000 mon.a (mon.0) 707 : cluster 1 overall HEALTH_OK 2024-04-13T07:30:01.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:00 smithi089 bash[31522]: audit 2024-04-13T07:30:00.261173+0000 mon.c (mon.2) 14 : audit 1 from='client.? 172.21.15.89:0/1525652961' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:01.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:00 smithi089 bash[31522]: audit 2024-04-13T07:30:00.261925+0000 mon.a (mon.0) 708 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:01.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:00 smithi089 bash[31522]: audit 2024-04-13T07:30:00.268540+0000 mon.a (mon.0) 709 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-13T07:30:02.049 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:01 smithi089 bash[56462]: ts=2024-04-13T07:30:01.568Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000817364s 2024-04-13T07:30:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:02 smithi203 bash[23480]: cluster 2024-04-13T07:30:00.851964+0000 mgr.y (mgr.14373) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:02 smithi089 bash[21353]: cluster 2024-04-13T07:30:00.851964+0000 mgr.y (mgr.14373) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:02.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:02 smithi089 bash[31522]: cluster 2024-04-13T07:30:00.851964+0000 mgr.y (mgr.14373) 32 : cluster 0 pgmap v14: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:04 smithi203 bash[23480]: cluster 2024-04-13T07:30:02.852666+0000 mgr.y (mgr.14373) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:04.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:04 smithi203 bash[23480]: audit 2024-04-13T07:30:03.229330+0000 mon.a (mon.0) 710 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:04.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:04 smithi203 bash[23480]: audit 2024-04-13T07:30:03.238913+0000 mon.a (mon.0) 711 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:04.517 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[31522]: cluster 2024-04-13T07:30:02.852666+0000 mgr.y (mgr.14373) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:04.517 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[31522]: audit 2024-04-13T07:30:03.229330+0000 mon.a (mon.0) 710 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:04.517 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[31522]: audit 2024-04-13T07:30:03.238913+0000 mon.a (mon.0) 711 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:04.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21353]: cluster 2024-04-13T07:30:02.852666+0000 mgr.y (mgr.14373) 33 : cluster 0 pgmap v15: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:04.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21353]: audit 2024-04-13T07:30:03.229330+0000 mon.a (mon.0) 710 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:04.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21353]: audit 2024-04-13T07:30:03.238913+0000 mon.a (mon.0) 711 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:04.518 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21606]: [13/Apr/2024:07:30:04] ENGINE Bus STOPPING 2024-04-13T07:30:04.518 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21606]: [13/Apr/2024:07:30:04] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-13T07:30:04.518 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21606]: [13/Apr/2024:07:30:04] ENGINE Bus STOPPED 2024-04-13T07:30:04.518 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21606]: [13/Apr/2024:07:30:04] ENGINE Bus STARTING 2024-04-13T07:30:04.518 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21606]: [13/Apr/2024:07:30:04] ENGINE Serving on http://:::9283 2024-04-13T07:30:04.714 INFO:teuthology.orchestra.run.smithi203.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.b/config 2024-04-13T07:30:04.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:04 smithi089 bash[21606]: [13/Apr/2024:07:30:04] ENGINE Bus STARTED 2024-04-13T07:30:05.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.277216+0000 mon.a (mon.0) 712 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.282828+0000 mon.a (mon.0) 713 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.283863+0000 mon.a (mon.0) 714 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.284632+0000 mon.a (mon.0) 715 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.289166+0000 mon.a (mon.0) 716 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: cephadm 2024-04-13T07:30:04.303047+0000 mgr.y (mgr.14373) 34 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.374284+0000 mon.a (mon.0) 717 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.381069+0000 mon.a (mon.0) 718 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.384426+0000 mon.a (mon.0) 719 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.385498+0000 mgr.y (mgr.14373) 35 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: audit 2024-04-13T07:30:04.395880+0000 mon.a (mon.0) 720 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[21353]: cephadm 2024-04-13T07:30:04.409026+0000 mgr.y (mgr.14373) 36 : cephadm 1 Deploying daemon grafana.a on smithi203 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.277216+0000 mon.a (mon.0) 712 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.282828+0000 mon.a (mon.0) 713 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.283863+0000 mon.a (mon.0) 714 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.284632+0000 mon.a (mon.0) 715 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.289166+0000 mon.a (mon.0) 716 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: cephadm 2024-04-13T07:30:04.303047+0000 mgr.y (mgr.14373) 34 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.374284+0000 mon.a (mon.0) 717 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.381069+0000 mon.a (mon.0) 718 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.384426+0000 mon.a (mon.0) 719 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.385498+0000 mgr.y (mgr.14373) 35 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: audit 2024-04-13T07:30:04.395880+0000 mon.a (mon.0) 720 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:05 smithi089 bash[31522]: cephadm 2024-04-13T07:30:04.409026+0000 mgr.y (mgr.14373) 36 : cephadm 1 Deploying daemon grafana.a on smithi203 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.277216+0000 mon.a (mon.0) 712 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.282828+0000 mon.a (mon.0) 713 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.283863+0000 mon.a (mon.0) 714 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.284632+0000 mon.a (mon.0) 715 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.289166+0000 mon.a (mon.0) 716 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: cephadm 2024-04-13T07:30:04.303047+0000 mgr.y (mgr.14373) 34 : cephadm 1 Regenerating cephadm self-signed grafana TLS certificates 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.374284+0000 mon.a (mon.0) 717 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.381069+0000 mon.a (mon.0) 718 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.384426+0000 mon.a (mon.0) 719 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.385498+0000 mgr.y (mgr.14373) 35 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2024-04-13T07:30:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: audit 2024-04-13T07:30:04.395880+0000 mon.a (mon.0) 720 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:05.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:05 smithi203 bash[23480]: cephadm 2024-04-13T07:30:04.409026+0000 mgr.y (mgr.14373) 36 : cephadm 1 Deploying daemon grafana.a on smithi203 2024-04-13T07:30:06.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:06 smithi203 bash[23480]: cluster 2024-04-13T07:30:04.853274+0000 mgr.y (mgr.14373) 37 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:06.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:06 smithi089 bash[21353]: cluster 2024-04-13T07:30:04.853274+0000 mgr.y (mgr.14373) 37 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:06.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:06 smithi089 bash[31522]: cluster 2024-04-13T07:30:04.853274+0000 mgr.y (mgr.14373) 37 : cluster 0 pgmap v16: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:08.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:08 smithi203 bash[23480]: cluster 2024-04-13T07:30:06.853921+0000 mgr.y (mgr.14373) 38 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:08.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:08 smithi089 bash[31522]: cluster 2024-04-13T07:30:06.853921+0000 mgr.y (mgr.14373) 38 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:08.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:08 smithi089 bash[21353]: cluster 2024-04-13T07:30:06.853921+0000 mgr.y (mgr.14373) 38 : cluster 0 pgmap v17: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:09.598 INFO:teuthology.orchestra.run.smithi203.stdout:[client.1] 2024-04-13T07:30:09.599 INFO:teuthology.orchestra.run.smithi203.stdout: key = AQCBNBpmeX+WIxAATzZ00U6XLYdnErFqCyoBhw== 2024-04-13T07:30:09.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:09 smithi203 bash[23480]: audit 2024-04-13T07:30:08.925431+0000 mon.a (mon.0) 721 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:09.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:09 smithi089 bash[21353]: audit 2024-04-13T07:30:08.925431+0000 mon.a (mon.0) 721 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:09.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:09 smithi089 bash[31522]: audit 2024-04-13T07:30:08.925431+0000 mon.a (mon.0) 721 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:09.799 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:09 smithi089 bash[56462]: ts=2024-04-13T07:30:09.569Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.0025498s 2024-04-13T07:30:10.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:10 smithi203 bash[23480]: cluster 2024-04-13T07:30:08.854464+0000 mgr.y (mgr.14373) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:10.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:10 smithi203 bash[23480]: audit 2024-04-13T07:30:09.595497+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.203:0/3478794104' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:10.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:10 smithi203 bash[23480]: audit 2024-04-13T07:30:09.596744+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:10.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:10 smithi203 bash[23480]: audit 2024-04-13T07:30:09.604221+0000 mon.a (mon.0) 723 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-13T07:30:10.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:10 smithi089 bash[21353]: cluster 2024-04-13T07:30:08.854464+0000 mgr.y (mgr.14373) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:10.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:10 smithi089 bash[21353]: audit 2024-04-13T07:30:09.595497+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.203:0/3478794104' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:10.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:10 smithi089 bash[21353]: audit 2024-04-13T07:30:09.596744+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:10.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:10 smithi089 bash[21353]: audit 2024-04-13T07:30:09.604221+0000 mon.a (mon.0) 723 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-13T07:30:10.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:10 smithi089 bash[31522]: cluster 2024-04-13T07:30:08.854464+0000 mgr.y (mgr.14373) 39 : cluster 0 pgmap v18: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:10.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:10 smithi089 bash[31522]: audit 2024-04-13T07:30:09.595497+0000 mon.b (mon.1) 29 : audit 1 from='client.? 172.21.15.203:0/3478794104' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:10.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:10 smithi089 bash[31522]: audit 2024-04-13T07:30:09.596744+0000 mon.a (mon.0) 722 : audit 1 from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2024-04-13T07:30:10.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:10 smithi089 bash[31522]: audit 2024-04-13T07:30:09.604221+0000 mon.a (mon.0) 723 : audit 1 from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2024-04-13T07:30:12.115 DEBUG:teuthology.orchestra.run.smithi203:> set -ex 2024-04-13T07:30:12.115 DEBUG:teuthology.orchestra.run.smithi203:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2024-04-13T07:30:12.115 DEBUG:teuthology.orchestra.run.smithi203:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2024-04-13T07:30:12.137 INFO:tasks.cephadm:Waiting for OSDs to come up 2024-04-13T07:30:12.137 DEBUG:teuthology.orchestra.run.smithi089:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2024-04-13T07:30:12.440 INFO:teuthology.misc.health.smithi089.stdout: 2024-04-13T07:30:12.440 INFO:teuthology.misc.health.smithi089.stdout:{"epoch":60,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","created":"2024-04-13T07:19:16.936150+0000","modified":"2024-04-13T07:29:38.824712+0000","last_up_change":"2024-04-13T07:28:40.551327+0000","last_in_change":"2024-04-13T07:28:13.615937+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-13T07:24:24.430140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9de691f5-25ed-472d-b6dd-dc3908fa85c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6803","nonce":3872923960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6805","nonce":3872923960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6809","nonce":3872923960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6807","nonce":3872923960}]},"public_addr":"172.21.15.89:6803/3872923960","cluster_addr":"172.21.15.89:6805/3872923960","heartbeat_back_addr":"172.21.15.89:6809/3872923960","heartbeat_front_addr":"172.21.15.89:6807/3872923960","state":["exists","up"]},{"osd":1,"uuid":"ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":34,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6811","nonce":302139657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6813","nonce":302139657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6817","nonce":302139657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6815","nonce":302139657}]},"public_addr":"172.21.15.89:6811/302139657","cluster_addr":"172.21.15.89:6813/302139657","heartbeat_back_addr":"172.21.15.89:6817/302139657","heartbeat_front_addr":"172.21.15.89:6815/302139657","state":["exists","up"]},{"osd":2,"uuid":"f86d6aea-97d3-4d75-99d0-6e5781d7a990","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6819","nonce":46732397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6821","nonce":46732397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6825","nonce":46732397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6823","nonce":46732397}]},"public_addr":"172.21.15.89:6819/46732397","cluster_addr":"172.21.15.89:6821/46732397","heartbeat_back_addr":"172.21.15.89:6825/46732397","heartbeat_front_addr":"172.21.15.89:6823/46732397","state":["exists","up"]},{"osd":3,"uuid":"37c263d3-d642-4cfa-99da-5a33e1bef16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6827","nonce":3306593060}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6829","nonce":3306593060}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6833","nonce":3306593060}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6831","nonce":3306593060}]},"public_addr":"172.21.15.89:6827/3306593060","cluster_addr":"172.21.15.89:6829/3306593060","heartbeat_back_addr":"172.21.15.89:6833/3306593060","heartbeat_front_addr":"172.21.15.89:6831/3306593060","state":["exists","up"]},{"osd":4,"uuid":"c1befaa2-c9f5-4023-9909-0577509ce44d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6801","nonce":3982510878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6803","nonce":3982510878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6807","nonce":3982510878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6805","nonce":3982510878}]},"public_addr":"172.21.15.203:6801/3982510878","cluster_addr":"172.21.15.203:6803/3982510878","heartbeat_back_addr":"172.21.15.203:6807/3982510878","heartbeat_front_addr":"172.21.15.203:6805/3982510878","state":["exists","up"]},{"osd":5,"uuid":"b45987d4-fc0f-45ad-9f0d-49240d898bfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":42,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6809","nonce":209623809}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6811","nonce":209623809}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6815","nonce":209623809}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6813","nonce":209623809}]},"public_addr":"172.21.15.203:6809/209623809","cluster_addr":"172.21.15.203:6811/209623809","heartbeat_back_addr":"172.21.15.203:6815/209623809","heartbeat_front_addr":"172.21.15.203:6813/209623809","state":["exists","up"]},{"osd":6,"uuid":"91bbf36b-799c-49df-8fbf-42a958699642","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":50,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6817","nonce":2138665907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6819","nonce":2138665907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6823","nonce":2138665907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6821","nonce":2138665907}]},"public_addr":"172.21.15.203:6817/2138665907","cluster_addr":"172.21.15.203:6819/2138665907","heartbeat_back_addr":"172.21.15.203:6823/2138665907","heartbeat_front_addr":"172.21.15.203:6821/2138665907","state":["exists","up"]},{"osd":7,"uuid":"aa9fb5c4-06e4-45ca-85f8-f2a96be15086","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6825","nonce":3848298771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6827","nonce":3848298771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6831","nonce":3848298771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6829","nonce":3848298771}]},"public_addr":"172.21.15.203:6825/3848298771","cluster_addr":"172.21.15.203:6827/3848298771","heartbeat_back_addr":"172.21.15.203:6831/3848298771","heartbeat_front_addr":"172.21.15.203:6829/3848298771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:22:41.252314+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:23:31.382195+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:24:20.791478+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:10.117632+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:59.130696+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:26:51.940707+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:27:44.966139+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:28:38.869265+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3115322972":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1139384373":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6800/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6801/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/28202064":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/2936084513":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6801/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/808783885":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/3574204772":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/868096906":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2680580613":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6801/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2483652022":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1215893273":"2024-04-14T07:20:19.796707+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-13T07:30:12.457 DEBUG:teuthology.misc:8 of 8 OSDs are up 2024-04-13T07:30:12.457 INFO:tasks.cephadm:Creating RBD pool 2024-04-13T07:30:12.457 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- sudo ceph --cluster ceph osd pool create rbd 8 2024-04-13T07:30:12.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:12 smithi203 bash[23480]: cluster 2024-04-13T07:30:10.855222+0000 mgr.y (mgr.14373) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:12.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:12 smithi089 bash[21353]: cluster 2024-04-13T07:30:10.855222+0000 mgr.y (mgr.14373) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:12.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:12 smithi089 bash[31522]: cluster 2024-04-13T07:30:10.855222+0000 mgr.y (mgr.14373) 40 : cluster 0 pgmap v19: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:13.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:13 smithi203 bash[23480]: audit 2024-04-13T07:30:12.444001+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.89:0/2502519875' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:13.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:13 smithi089 bash[31522]: audit 2024-04-13T07:30:12.444001+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.89:0/2502519875' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:13.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:13 smithi089 bash[21353]: audit 2024-04-13T07:30:12.444001+0000 mon.c (mon.2) 15 : audit 0 from='client.? 172.21.15.89:0/2502519875' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:14.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:14 smithi203 bash[23480]: cluster 2024-04-13T07:30:12.855780+0000 mgr.y (mgr.14373) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:14.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:14 smithi089 bash[21353]: cluster 2024-04-13T07:30:12.855780+0000 mgr.y (mgr.14373) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:14.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:14 smithi089 bash[31522]: cluster 2024-04-13T07:30:12.855780+0000 mgr.y (mgr.14373) 41 : cluster 0 pgmap v20: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:16.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:16 smithi203 bash[23480]: cluster 2024-04-13T07:30:14.856361+0000 mgr.y (mgr.14373) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:16 smithi089 bash[31522]: cluster 2024-04-13T07:30:14.856361+0000 mgr.y (mgr.14373) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:16 smithi089 bash[21353]: cluster 2024-04-13T07:30:14.856361+0000 mgr.y (mgr.14373) 42 : cluster 0 pgmap v21: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:17.164 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:18.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:18 smithi089 bash[21353]: cluster 2024-04-13T07:30:16.857028+0000 mgr.y (mgr.14373) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:18.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:18 smithi089 bash[31522]: cluster 2024-04-13T07:30:16.857028+0000 mgr.y (mgr.14373) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:18.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:18 smithi203 bash[23480]: cluster 2024-04-13T07:30:16.857028+0000 mgr.y (mgr.14373) 43 : cluster 0 pgmap v22: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:20.450 INFO:teuthology.orchestra.run.smithi089.stderr:pool 'rbd' created 2024-04-13T07:30:20.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:20 smithi203 bash[23480]: cluster 2024-04-13T07:30:18.857619+0000 mgr.y (mgr.14373) 44 : cluster 0 pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:20.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:20 smithi203 bash[23480]: audit 2024-04-13T07:30:19.791549+0000 mon.a (mon.0) 724 : audit 1 from='client.? 172.21.15.89:0/1361707926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-13T07:30:20.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:20 smithi089 bash[31522]: cluster 2024-04-13T07:30:18.857619+0000 mgr.y (mgr.14373) 44 : cluster 0 pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:20.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:20 smithi089 bash[31522]: audit 2024-04-13T07:30:19.791549+0000 mon.a (mon.0) 724 : audit 1 from='client.? 172.21.15.89:0/1361707926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-13T07:30:20.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:20 smithi089 bash[21353]: cluster 2024-04-13T07:30:18.857619+0000 mgr.y (mgr.14373) 44 : cluster 0 pgmap v23: 1 pgs: 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:20.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:20 smithi089 bash[21353]: audit 2024-04-13T07:30:19.791549+0000 mon.a (mon.0) 724 : audit 1 from='client.? 172.21.15.89:0/1361707926' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2024-04-13T07:30:21.085 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2024-04-13T07:30:21.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:21 smithi203 bash[23480]: audit 2024-04-13T07:30:20.451437+0000 mon.a (mon.0) 725 : audit 1 from='client.? 172.21.15.89:0/1361707926' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-13T07:30:21.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:21 smithi203 bash[23480]: cluster 2024-04-13T07:30:20.451649+0000 mon.a (mon.0) 726 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-04-13T07:30:21.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:21 smithi089 bash[21353]: audit 2024-04-13T07:30:20.451437+0000 mon.a (mon.0) 725 : audit 1 from='client.? 172.21.15.89:0/1361707926' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-13T07:30:21.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:21 smithi089 bash[21353]: cluster 2024-04-13T07:30:20.451649+0000 mon.a (mon.0) 726 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-04-13T07:30:21.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:21 smithi089 bash[31522]: audit 2024-04-13T07:30:20.451437+0000 mon.a (mon.0) 725 : audit 1 from='client.? 172.21.15.89:0/1361707926' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2024-04-13T07:30:21.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:21 smithi089 bash[31522]: cluster 2024-04-13T07:30:20.451649+0000 mon.a (mon.0) 726 : cluster 0 osdmap e61: 8 total, 8 up, 8 in 2024-04-13T07:30:22.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:22 smithi203 bash[23480]: cluster 2024-04-13T07:30:20.858290+0000 mgr.y (mgr.14373) 45 : cluster 0 pgmap v25: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:22.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:22 smithi203 bash[23480]: cluster 2024-04-13T07:30:21.445417+0000 mon.a (mon.0) 727 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-13T07:30:22.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:22 smithi203 bash[23480]: cluster 2024-04-13T07:30:21.461649+0000 mon.a (mon.0) 728 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-04-13T07:30:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:22 smithi089 bash[21353]: cluster 2024-04-13T07:30:20.858290+0000 mgr.y (mgr.14373) 45 : cluster 0 pgmap v25: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:22 smithi089 bash[21353]: cluster 2024-04-13T07:30:21.445417+0000 mon.a (mon.0) 727 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-13T07:30:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:22 smithi089 bash[21353]: cluster 2024-04-13T07:30:21.461649+0000 mon.a (mon.0) 728 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-04-13T07:30:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:22 smithi089 bash[31522]: cluster 2024-04-13T07:30:20.858290+0000 mgr.y (mgr.14373) 45 : cluster 0 pgmap v25: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:22 smithi089 bash[31522]: cluster 2024-04-13T07:30:21.445417+0000 mon.a (mon.0) 727 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-13T07:30:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:22 smithi089 bash[31522]: cluster 2024-04-13T07:30:21.461649+0000 mon.a (mon.0) 728 : cluster 0 osdmap e62: 8 total, 8 up, 8 in 2024-04-13T07:30:23.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:23 smithi203 bash[23480]: cluster 2024-04-13T07:30:22.469792+0000 mon.a (mon.0) 729 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-04-13T07:30:23.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:23 smithi089 bash[31522]: cluster 2024-04-13T07:30:22.469792+0000 mon.a (mon.0) 729 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-04-13T07:30:23.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:23 smithi089 bash[21353]: cluster 2024-04-13T07:30:22.469792+0000 mon.a (mon.0) 729 : cluster 0 osdmap e63: 8 total, 8 up, 8 in 2024-04-13T07:30:24.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:24 smithi203 bash[23480]: cluster 2024-04-13T07:30:22.858954+0000 mgr.y (mgr.14373) 46 : cluster 0 pgmap v28: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:24.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:24 smithi203 bash[23480]: audit 2024-04-13T07:30:23.932944+0000 mon.a (mon.0) 730 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:24.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:24 smithi203 bash[23480]: audit 2024-04-13T07:30:23.934730+0000 mon.a (mon.0) 731 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:24 smithi089 bash[21353]: cluster 2024-04-13T07:30:22.858954+0000 mgr.y (mgr.14373) 46 : cluster 0 pgmap v28: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:24 smithi089 bash[21353]: audit 2024-04-13T07:30:23.932944+0000 mon.a (mon.0) 730 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:24 smithi089 bash[21353]: audit 2024-04-13T07:30:23.934730+0000 mon.a (mon.0) 731 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:24.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:24 smithi089 bash[31522]: cluster 2024-04-13T07:30:22.858954+0000 mgr.y (mgr.14373) 46 : cluster 0 pgmap v28: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:24.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:24 smithi089 bash[31522]: audit 2024-04-13T07:30:23.932944+0000 mon.a (mon.0) 730 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:24.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:24 smithi089 bash[31522]: audit 2024-04-13T07:30:23.934730+0000 mon.a (mon.0) 731 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:25.816 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:26.751 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:26 smithi203 bash[23480]: cluster 2024-04-13T07:30:24.859579+0000 mgr.y (mgr.14373) 47 : cluster 0 pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:26.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:26 smithi089 bash[31522]: cluster 2024-04-13T07:30:24.859579+0000 mgr.y (mgr.14373) 47 : cluster 0 pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:26.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:26 smithi089 bash[21353]: cluster 2024-04-13T07:30:24.859579+0000 mgr.y (mgr.14373) 47 : cluster 0 pgmap v29: 9 pgs: 8 unknown, 1 active+clean; 577 KiB data, 229 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:28.500 INFO:teuthology.orchestra.run.smithi089.stderr:enabled application 'rbd' on pool 'rbd' 2024-04-13T07:30:28.656 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:28 smithi203 bash[23480]: cluster 2024-04-13T07:30:26.860540+0000 mgr.y (mgr.14373) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:28.656 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:28 smithi203 bash[23480]: audit 2024-04-13T07:30:28.269292+0000 mon.a (mon.0) 732 : audit 1 from='client.? 172.21.15.89:0/1987451170' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:30:28.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:28 smithi089 bash[31522]: cluster 2024-04-13T07:30:26.860540+0000 mgr.y (mgr.14373) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:28.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:28 smithi089 bash[31522]: audit 2024-04-13T07:30:28.269292+0000 mon.a (mon.0) 732 : audit 1 from='client.? 172.21.15.89:0/1987451170' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:30:28.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:28 smithi089 bash[21353]: cluster 2024-04-13T07:30:26.860540+0000 mgr.y (mgr.14373) 48 : cluster 0 pgmap v30: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:28.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:28 smithi089 bash[21353]: audit 2024-04-13T07:30:28.269292+0000 mon.a (mon.0) 732 : audit 1 from='client.? 172.21.15.89:0/1987451170' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2024-04-13T07:30:29.105 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-13T07:30:29.106 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-13T07:30:29.106 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph mgr dump --format=json 2024-04-13T07:30:29.585 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.586 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.586 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.586 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.587 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.587 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.587 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.587 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.587 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:29 smithi203 bash[23480]: audit 2024-04-13T07:30:28.500951+0000 mon.a (mon.0) 733 : audit 1 from='client.? 172.21.15.89:0/1987451170' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-13T07:30:29.588 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:29 smithi203 bash[23480]: cluster 2024-04-13T07:30:28.501196+0000 mon.a (mon.0) 734 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-04-13T07:30:29.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:29 smithi089 bash[21353]: audit 2024-04-13T07:30:28.500951+0000 mon.a (mon.0) 733 : audit 1 from='client.? 172.21.15.89:0/1987451170' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-13T07:30:29.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:29 smithi089 bash[21353]: cluster 2024-04-13T07:30:28.501196+0000 mon.a (mon.0) 734 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-04-13T07:30:29.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:29 smithi089 bash[31522]: audit 2024-04-13T07:30:28.500951+0000 mon.a (mon.0) 733 : audit 1 from='client.? 172.21.15.89:0/1987451170' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2024-04-13T07:30:29.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:29 smithi089 bash[31522]: cluster 2024-04-13T07:30:28.501196+0000 mon.a (mon.0) 734 : cluster 0 osdmap e64: 8 total, 8 up, 8 in 2024-04-13T07:30:29.843 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.843 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.844 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.844 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.844 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.845 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.845 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.845 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:29.845 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: Started Ceph grafana.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:30:29.845 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:29 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:30:30.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: cluster 2024-04-13T07:30:28.861184+0000 mgr.y (mgr.14373) 49 : cluster 0 pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:30.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: cluster 2024-04-13T07:30:29.506855+0000 mon.a (mon.0) 735 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-13T07:30:30.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: cluster 2024-04-13T07:30:29.506915+0000 mon.a (mon.0) 736 : cluster 1 Cluster is now healthy 2024-04-13T07:30:30.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: audit 2024-04-13T07:30:29.890251+0000 mon.a (mon.0) 737 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: audit 2024-04-13T07:30:29.902683+0000 mon.a (mon.0) 738 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: audit 2024-04-13T07:30:29.920220+0000 mon.a (mon.0) 739 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: audit 2024-04-13T07:30:29.929473+0000 mon.a (mon.0) 740 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: audit 2024-04-13T07:30:29.996663+0000 mon.a (mon.0) 741 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:30:30.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: audit 2024-04-13T07:30:30.371715+0000 mon.a (mon.0) 742 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:30 smithi203 bash[23480]: audit 2024-04-13T07:30:30.378806+0000 mon.a (mon.0) 743 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: cluster 2024-04-13T07:30:28.861184+0000 mgr.y (mgr.14373) 49 : cluster 0 pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: cluster 2024-04-13T07:30:29.506855+0000 mon.a (mon.0) 735 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-13T07:30:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: cluster 2024-04-13T07:30:29.506915+0000 mon.a (mon.0) 736 : cluster 1 Cluster is now healthy 2024-04-13T07:30:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: audit 2024-04-13T07:30:29.890251+0000 mon.a (mon.0) 737 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: audit 2024-04-13T07:30:29.902683+0000 mon.a (mon.0) 738 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: audit 2024-04-13T07:30:29.920220+0000 mon.a (mon.0) 739 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: audit 2024-04-13T07:30:29.929473+0000 mon.a (mon.0) 740 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: audit 2024-04-13T07:30:29.996663+0000 mon.a (mon.0) 741 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: audit 2024-04-13T07:30:30.371715+0000 mon.a (mon.0) 742 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[21353]: audit 2024-04-13T07:30:30.378806+0000 mon.a (mon.0) 743 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: cluster 2024-04-13T07:30:28.861184+0000 mgr.y (mgr.14373) 49 : cluster 0 pgmap v32: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: cluster 2024-04-13T07:30:29.506855+0000 mon.a (mon.0) 735 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: cluster 2024-04-13T07:30:29.506915+0000 mon.a (mon.0) 736 : cluster 1 Cluster is now healthy 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: audit 2024-04-13T07:30:29.890251+0000 mon.a (mon.0) 737 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: audit 2024-04-13T07:30:29.902683+0000 mon.a (mon.0) 738 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: audit 2024-04-13T07:30:29.920220+0000 mon.a (mon.0) 739 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: audit 2024-04-13T07:30:29.929473+0000 mon.a (mon.0) 740 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: audit 2024-04-13T07:30:29.996663+0000 mon.a (mon.0) 741 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:30:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: audit 2024-04-13T07:30:30.371715+0000 mon.a (mon.0) 742 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:30.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:30 smithi089 bash[31522]: audit 2024-04-13T07:30:30.378806+0000 mon.a (mon.0) 743 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:31.229 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193504608Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193694991Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193715541Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193728466Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193736546Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.1937436Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193750443Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193757738Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193766203Z level=info msg="Path Home" path=/usr/share/grafana 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193785161Z level=info msg="Path Data" path=/var/lib/grafana 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193794534Z level=info msg="Path Logs" path=/var/log/grafana 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193803604Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2024-04-13T07:30:31.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193814216Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2024-04-13T07:30:31.231 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=settings t=2024-04-13T07:30:31.193825206Z level=info msg="App mode production" 2024-04-13T07:30:31.231 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=sqlstore t=2024-04-13T07:30:31.19389901Z level=info msg="Connecting to DB" dbtype=sqlite3 2024-04-13T07:30:31.231 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=sqlstore t=2024-04-13T07:30:31.193918642Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2024-04-13T07:30:31.231 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.194855958Z level=info msg="Starting DB migrations" 2024-04-13T07:30:31.231 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.194937474Z level=info msg="Executing migration" id="create migration_log table" 2024-04-13T07:30:31.231 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.234714847Z level=info msg="Executing migration" id="create user table" 2024-04-13T07:30:31.489 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.259893963Z level=info msg="Executing migration" id="add unique index user.login" 2024-04-13T07:30:31.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.301933988Z level=info msg="Executing migration" id="add unique index user.email" 2024-04-13T07:30:31.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.344086024Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2024-04-13T07:30:31.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.369170471Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2024-04-13T07:30:31.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.394508872Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2024-04-13T07:30:31.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.419703926Z level=info msg="Executing migration" id="create user table v2" 2024-04-13T07:30:31.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.444801593Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2024-04-13T07:30:31.490 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.470178286Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2024-04-13T07:30:31.742 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.495456336Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2024-04-13T07:30:31.742 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.537313334Z level=info msg="Executing migration" id="Drop old table user_v1" 2024-04-13T07:30:31.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.57959633Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2024-04-13T07:30:31.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.604736973Z level=info msg="Executing migration" id="Update user table charset" 2024-04-13T07:30:31.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.63007012Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2024-04-13T07:30:31.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.672011206Z level=info msg="Executing migration" id="Add missing user data" 2024-04-13T07:30:31.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.713959929Z level=info msg="Executing migration" id="Add is_disabled column to user" 2024-04-13T07:30:31.743 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.74786859Z level=info msg="Executing migration" id="Add index user.login/user.email" 2024-04-13T07:30:32.093 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.773062954Z level=info msg="Executing migration" id="Add is_service_account column to user" 2024-04-13T07:30:32.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.798310995Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2024-04-13T07:30:32.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.823512411Z level=info msg="Executing migration" id="create temp user table v1-7" 2024-04-13T07:30:32.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.848894411Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2024-04-13T07:30:32.094 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:31 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:31.874136266Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2024-04-13T07:30:32.346 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.099560829Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2024-04-13T07:30:32.346 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.124750504Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2024-04-13T07:30:32.346 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.166709588Z level=info msg="Executing migration" id="Update temp_user table charset" 2024-04-13T07:30:32.346 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.191977133Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2024-04-13T07:30:32.346 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.225550643Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2024-04-13T07:30:32.346 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.284286933Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2024-04-13T07:30:32.346 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.317918164Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2024-04-13T07:30:32.347 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.351496249Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2024-04-13T07:30:32.623 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[23480]: cluster 2024-04-13T07:30:30.861875+0000 mgr.y (mgr.14373) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:32.623 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.38506592Z level=info msg="Executing migration" id="create temp_user v2" 2024-04-13T07:30:32.623 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.418825772Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2024-04-13T07:30:32.624 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.444038178Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2024-04-13T07:30:32.624 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.477666343Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2024-04-13T07:30:32.624 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.511229356Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2024-04-13T07:30:32.624 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.561778797Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2024-04-13T07:30:32.624 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.595234288Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2024-04-13T07:30:32.624 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.628853981Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2024-04-13T07:30:32.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:32 smithi089 bash[21353]: cluster 2024-04-13T07:30:30.861875+0000 mgr.y (mgr.14373) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:32.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:32 smithi089 bash[31522]: cluster 2024-04-13T07:30:30.861875+0000 mgr.y (mgr.14373) 50 : cluster 0 pgmap v33: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:32.883 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.682733129Z level=info msg="Executing migration" id="create star table" 2024-04-13T07:30:32.883 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.737823346Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2024-04-13T07:30:32.883 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.771618036Z level=info msg="Executing migration" id="create org table v1" 2024-04-13T07:30:32.883 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.821911769Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2024-04-13T07:30:32.883 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.85527789Z level=info msg="Executing migration" id="create org_user table v1" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.889139101Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.93123116Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:32 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:32.981620188Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.006930497Z level=info msg="Executing migration" id="Update org table charset" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.032066692Z level=info msg="Executing migration" id="Update org_user table charset" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.05736075Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.082655532Z level=info msg="Executing migration" id="create dashboard table" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.107928274Z level=info msg="Executing migration" id="add index dashboard.account_id" 2024-04-13T07:30:33.145 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.149965517Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2024-04-13T07:30:33.406 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.191959667Z level=info msg="Executing migration" id="create dashboard_tag table" 2024-04-13T07:30:33.406 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.217133921Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2024-04-13T07:30:33.406 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.259166743Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2024-04-13T07:30:33.406 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.284435299Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2024-04-13T07:30:33.406 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.326387723Z level=info msg="Executing migration" id="create dashboard v2" 2024-04-13T07:30:33.406 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.351711431Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2024-04-13T07:30:33.406 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.377011776Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2024-04-13T07:30:33.406 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.394060323Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2024-04-13T07:30:33.407 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.410854236Z level=info msg="Executing migration" id="drop table dashboard_v1" 2024-04-13T07:30:33.657 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.436335652Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2024-04-13T07:30:33.657 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.461308779Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2024-04-13T07:30:33.658 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.486585875Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2024-04-13T07:30:33.658 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.511823428Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2024-04-13T07:30:33.658 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.553785875Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2024-04-13T07:30:33.658 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.595852037Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2024-04-13T07:30:33.658 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.621079926Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2024-04-13T07:30:33.658 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.663017994Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2024-04-13T07:30:33.828 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.704883232Z level=info msg="Executing migration" id="Update dashboard table charset" 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.730232476Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.75539681Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.780768934Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.806234071Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.831468507Z level=info msg="Executing migration" id="Add column uid in dashboard" 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.85669051Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.881955289Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2024-04-13T07:30:33.919 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.923980569Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2024-04-13T07:30:34.179 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.949294617Z level=info msg="Executing migration" id="Update dashboard title length" 2024-04-13T07:30:34.179 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:33 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.974560576Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2024-04-13T07:30:34.179 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:33.999832545Z level=info msg="Executing migration" id="create dashboard_provisioning" 2024-04-13T07:30:34.179 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.025134834Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2024-04-13T07:30:34.179 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.050293404Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2024-04-13T07:30:34.179 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.075597583Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2024-04-13T07:30:34.180 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.117507912Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2024-04-13T07:30:34.180 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.142786555Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2024-04-13T07:30:34.180 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.184659054Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2024-04-13T07:30:34.439 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.226722159Z level=info msg="Executing migration" id="Add check_sum column" 2024-04-13T07:30:34.461 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.277063717Z level=info msg="Executing migration" id="Add index for dashboard_title" 2024-04-13T07:30:34.461 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.30240284Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2024-04-13T07:30:34.461 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.327711521Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2024-04-13T07:30:34.461 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.352919062Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2024-04-13T07:30:34.461 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.37808916Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2024-04-13T07:30:34.461 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.411809236Z level=info msg="Executing migration" id="create data_source table" 2024-04-13T07:30:34.692 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[23480]: cluster 2024-04-13T07:30:32.862327+0000 mgr.y (mgr.14373) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:34.692 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[23480]: audit 2024-04-13T07:30:34.276106+0000 mon.a (mon.0) 744 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:34.692 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.445326936Z level=info msg="Executing migration" id="add index data_source.account_id" 2024-04-13T07:30:34.692 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.478972795Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2024-04-13T07:30:34.692 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.512605623Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2024-04-13T07:30:34.692 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.563037777Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2024-04-13T07:30:34.692 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.613315592Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2024-04-13T07:30:34.693 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.663771542Z level=info msg="Executing migration" id="create data_source table v2" 2024-04-13T07:30:34.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:34 smithi089 bash[31522]: cluster 2024-04-13T07:30:32.862327+0000 mgr.y (mgr.14373) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:34.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:34 smithi089 bash[31522]: audit 2024-04-13T07:30:34.276106+0000 mon.a (mon.0) 744 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:34.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:34 smithi089 bash[21353]: cluster 2024-04-13T07:30:32.862327+0000 mgr.y (mgr.14373) 51 : cluster 0 pgmap v34: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:34.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:34 smithi089 bash[21353]: audit 2024-04-13T07:30:34.276106+0000 mon.a (mon.0) 744 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:34.977 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.697417755Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2024-04-13T07:30:34.977 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.7308961Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2024-04-13T07:30:34.978 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.773228389Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2024-04-13T07:30:34.978 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.806773508Z level=info msg="Executing migration" id="Add column with_credentials" 2024-04-13T07:30:34.978 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.84028708Z level=info msg="Executing migration" id="Add secure json data column" 2024-04-13T07:30:34.978 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.873941033Z level=info msg="Executing migration" id="Update data_source table charset" 2024-04-13T07:30:34.978 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.915867111Z level=info msg="Executing migration" id="Update initial version to 1" 2024-04-13T07:30:34.978 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.941287692Z level=info msg="Executing migration" id="Add read_only data column" 2024-04-13T07:30:34.978 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:34 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:34.983232803Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2024-04-13T07:30:35.229 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.025435675Z level=info msg="Executing migration" id="Update json_data with nulls" 2024-04-13T07:30:35.229 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.050741043Z level=info msg="Executing migration" id="Add uid column" 2024-04-13T07:30:35.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.092598011Z level=info msg="Executing migration" id="Update uid value" 2024-04-13T07:30:35.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.167895368Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2024-04-13T07:30:35.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.193201408Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2024-04-13T07:30:35.230 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.235283635Z level=info msg="Executing migration" id="create api_key table" 2024-04-13T07:30:35.482 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.285658815Z level=info msg="Executing migration" id="add index api_key.account_id" 2024-04-13T07:30:35.482 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.310993605Z level=info msg="Executing migration" id="add index api_key.key" 2024-04-13T07:30:35.482 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.353020927Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2024-04-13T07:30:35.482 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.445371078Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2024-04-13T07:30:35.751 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.487518767Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2024-04-13T07:30:35.751 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.512845425Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2024-04-13T07:30:35.751 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.554829354Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2024-04-13T07:30:35.751 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.580101415Z level=info msg="Executing migration" id="create api_key table v2" 2024-04-13T07:30:35.751 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.664145752Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2024-04-13T07:30:35.751 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.714742584Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2024-04-13T07:30:35.751 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.756740019Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2024-04-13T07:30:36.016 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.781949325Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2024-04-13T07:30:36.016 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.807113356Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2024-04-13T07:30:36.016 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.832377716Z level=info msg="Executing migration" id="Update api_key table charset" 2024-04-13T07:30:36.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.874341108Z level=info msg="Executing migration" id="Add expires to api_key table" 2024-04-13T07:30:36.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.93318108Z level=info msg="Executing migration" id="Add service account foreign key" 2024-04-13T07:30:36.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.958480039Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2024-04-13T07:30:36.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:35 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:35.983760399Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2024-04-13T07:30:36.281 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:30:36.289 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.025722404Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2024-04-13T07:30:36.289 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.051002714Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2024-04-13T07:30:36.289 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.076189729Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2024-04-13T07:30:36.289 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.101452777Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2024-04-13T07:30:36.290 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.143444348Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2024-04-13T07:30:36.290 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.185225377Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2024-04-13T07:30:36.290 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.227438808Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2024-04-13T07:30:36.290 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.269306447Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2024-04-13T07:30:36.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:30:36] "GET /metrics HTTP/1.1" 200 28563 "" "Prometheus/2.43.0" 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: cluster 2024-04-13T07:30:34.862731+0000 mgr.y (mgr.14373) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: audit 2024-04-13T07:30:35.412326+0000 mon.a (mon.0) 745 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: audit 2024-04-13T07:30:35.417060+0000 mon.a (mon.0) 746 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: audit 2024-04-13T07:30:35.689663+0000 mon.a (mon.0) 747 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: audit 2024-04-13T07:30:35.701441+0000 mon.a (mon.0) 748 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: audit 2024-04-13T07:30:35.703903+0000 mon.a (mon.0) 749 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: audit 2024-04-13T07:30:35.705631+0000 mon.a (mon.0) 750 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: audit 2024-04-13T07:30:35.716431+0000 mon.a (mon.0) 751 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: cephadm 2024-04-13T07:30:35.792096+0000 mgr.y (mgr.14373) 53 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: cephadm 2024-04-13T07:30:35.797473+0000 mgr.y (mgr.14373) 54 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi089 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[31522]: audit 2024-04-13T07:30:36.277260+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.89:0/2506381170' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: cluster 2024-04-13T07:30:34.862731+0000 mgr.y (mgr.14373) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: audit 2024-04-13T07:30:35.412326+0000 mon.a (mon.0) 745 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: audit 2024-04-13T07:30:35.417060+0000 mon.a (mon.0) 746 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: audit 2024-04-13T07:30:35.689663+0000 mon.a (mon.0) 747 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: audit 2024-04-13T07:30:35.701441+0000 mon.a (mon.0) 748 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: audit 2024-04-13T07:30:35.703903+0000 mon.a (mon.0) 749 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: audit 2024-04-13T07:30:35.705631+0000 mon.a (mon.0) 750 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: audit 2024-04-13T07:30:35.716431+0000 mon.a (mon.0) 751 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: cephadm 2024-04-13T07:30:35.792096+0000 mgr.y (mgr.14373) 53 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-04-13T07:30:36.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: cephadm 2024-04-13T07:30:35.797473+0000 mgr.y (mgr.14373) 54 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi089 2024-04-13T07:30:36.552 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:36 smithi089 bash[21353]: audit 2024-04-13T07:30:36.277260+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.89:0/2506381170' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:30:36.566 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.294518142Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2024-04-13T07:30:36.566 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.336486736Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2024-04-13T07:30:36.566 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.378513833Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2024-04-13T07:30:36.566 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.403779761Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2024-04-13T07:30:36.567 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.445917252Z level=info msg="Executing migration" id="create quota table v1" 2024-04-13T07:30:36.567 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.471216143Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2024-04-13T07:30:36.567 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.496424131Z level=info msg="Executing migration" id="Update quota table charset" 2024-04-13T07:30:36.567 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.53835382Z level=info msg="Executing migration" id="create plugin_setting table" 2024-04-13T07:30:36.567 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.572037681Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: cluster 2024-04-13T07:30:34.862731+0000 mgr.y (mgr.14373) 52 : cluster 0 pgmap v35: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: audit 2024-04-13T07:30:35.412326+0000 mon.a (mon.0) 745 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: audit 2024-04-13T07:30:35.417060+0000 mon.a (mon.0) 746 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: audit 2024-04-13T07:30:35.689663+0000 mon.a (mon.0) 747 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: audit 2024-04-13T07:30:35.701441+0000 mon.a (mon.0) 748 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: audit 2024-04-13T07:30:35.703903+0000 mon.a (mon.0) 749 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: audit 2024-04-13T07:30:35.705631+0000 mon.a (mon.0) 750 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: audit 2024-04-13T07:30:35.716431+0000 mon.a (mon.0) 751 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:36.567 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: cephadm 2024-04-13T07:30:35.792096+0000 mgr.y (mgr.14373) 53 : cephadm 1 Reconfiguring alertmanager.a (dependencies changed)... 2024-04-13T07:30:36.568 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: cephadm 2024-04-13T07:30:35.797473+0000 mgr.y (mgr.14373) 54 : cephadm 1 Reconfiguring daemon alertmanager.a on smithi089 2024-04-13T07:30:36.568 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[23480]: audit 2024-04-13T07:30:36.277260+0000 mon.c (mon.2) 16 : audit 0 from='client.? 172.21.15.89:0/2506381170' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:30:36.819 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.588952207Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2024-04-13T07:30:36.819 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.622535058Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2024-04-13T07:30:36.819 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.656038501Z level=info msg="Executing migration" id="create session table" 2024-04-13T07:30:36.820 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.69011084Z level=info msg="Executing migration" id="Drop old table playlist table" 2024-04-13T07:30:36.820 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.723685792Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2024-04-13T07:30:36.820 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.75725136Z level=info msg="Executing migration" id="create playlist table v2" 2024-04-13T07:30:36.820 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.790893401Z level=info msg="Executing migration" id="create playlist item table v2" 2024-04-13T07:30:36.820 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.824347837Z level=info msg="Executing migration" id="Update playlist table charset" 2024-04-13T07:30:37.071 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.858057971Z level=info msg="Executing migration" id="Update playlist_item table charset" 2024-04-13T07:30:37.071 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.891652699Z level=info msg="Executing migration" id="drop preferences table v2" 2024-04-13T07:30:37.071 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.925217244Z level=info msg="Executing migration" id="drop preferences table v3" 2024-04-13T07:30:37.071 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.950493337Z level=info msg="Executing migration" id="create preferences table v3" 2024-04-13T07:30:37.071 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:36 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:36.975780187Z level=info msg="Executing migration" id="Update preferences table charset" 2024-04-13T07:30:37.071 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.001000632Z level=info msg="Executing migration" id="Add column team_id in preferences" 2024-04-13T07:30:37.071 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.026271056Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2024-04-13T07:30:37.072 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.051561843Z level=info msg="Executing migration" id="Add column week_start in preferences" 2024-04-13T07:30:37.072 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.076812777Z level=info msg="Executing migration" id="Add column preferences.json_data" 2024-04-13T07:30:37.193 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":20,"flags":0,"active_gid":14373,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":2897907290},{"type":"v1","addr":"172.21.15.89:6801","nonce":2897907290}]},"active_addr":"172.21.15.89:6801/2897907290","active_change":"2024-04-13T07:29:38.824930+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24286,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3991952503}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1853920825}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":4124851388}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":4260385376}]}]} 2024-04-13T07:30:37.197 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-13T07:30:37.197 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-13T07:30:37.198 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-13T07:30:37.332 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.101995459Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2024-04-13T07:30:37.332 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.127294249Z level=info msg="Executing migration" id="Add preferences index org_id" 2024-04-13T07:30:37.332 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.16935761Z level=info msg="Executing migration" id="Add preferences index user_id" 2024-04-13T07:30:37.332 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.194747082Z level=info msg="Executing migration" id="create alert table v1" 2024-04-13T07:30:37.332 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.23659711Z level=info msg="Executing migration" id="add index alert org_id & id " 2024-04-13T07:30:37.333 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.262016135Z level=info msg="Executing migration" id="add index alert state" 2024-04-13T07:30:37.333 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.287102521Z level=info msg="Executing migration" id="add index alert dashboard_id" 2024-04-13T07:30:37.333 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.312285577Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2024-04-13T07:30:37.333 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.337587737Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2024-04-13T07:30:37.618 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.379551773Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2024-04-13T07:30:37.618 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.404827387Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2024-04-13T07:30:37.618 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.44676106Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2024-04-13T07:30:37.618 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.488760862Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2024-04-13T07:30:37.618 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.514110089Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2024-04-13T07:30:37.619 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.556004203Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2024-04-13T07:30:37.619 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.581323749Z level=info msg="Executing migration" id="create alert_notification table v1" 2024-04-13T07:30:37.619 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.623488149Z level=info msg="Executing migration" id="Add column is_default" 2024-04-13T07:30:37.870 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.657102124Z level=info msg="Executing migration" id="Add column frequency" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.674100683Z level=info msg="Executing migration" id="Add column send_reminder" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.690986593Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.707893106Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.724811922Z level=info msg="Executing migration" id="Update alert table charset" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.750009587Z level=info msg="Executing migration" id="Update alert_notification table charset" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.775227801Z level=info msg="Executing migration" id="create notification_journal table v1" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.800535738Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.825781729Z level=info msg="Executing migration" id="drop alert_notification_journal" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.85097807Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2024-04-13T07:30:37.871 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.876139289Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2024-04-13T07:30:38.156 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.901433769Z level=info msg="Executing migration" id="Add for to alert table" 2024-04-13T07:30:38.157 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.926667364Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2024-04-13T07:30:38.157 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.951916228Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2024-04-13T07:30:38.157 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:37 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:37.977219705Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2024-04-13T07:30:38.157 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.002476566Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2024-04-13T07:30:38.157 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.027720877Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2024-04-13T07:30:38.157 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.069618711Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2024-04-13T07:30:38.157 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.094927521Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2024-04-13T07:30:38.157 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.120110487Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2024-04-13T07:30:38.158 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.162152436Z level=info msg="Executing migration" id="Drop old annotation table v4" 2024-04-13T07:30:38.410 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.204058471Z level=info msg="Executing migration" id="create annotation table v5" 2024-04-13T07:30:38.411 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.246127282Z level=info msg="Executing migration" id="add index annotation 0 v3" 2024-04-13T07:30:38.411 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.288064257Z level=info msg="Executing migration" id="add index annotation 1 v3" 2024-04-13T07:30:38.411 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.330171775Z level=info msg="Executing migration" id="add index annotation 2 v3" 2024-04-13T07:30:38.411 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.355330894Z level=info msg="Executing migration" id="add index annotation 3 v3" 2024-04-13T07:30:38.411 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.380666614Z level=info msg="Executing migration" id="add index annotation 4 v3" 2024-04-13T07:30:38.411 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.405904392Z level=info msg="Executing migration" id="Update annotation table charset" 2024-04-13T07:30:38.703 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[23480]: cluster 2024-04-13T07:30:36.863411+0000 mgr.y (mgr.14373) 55 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:38.703 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.447980664Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2024-04-13T07:30:38.703 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.481844344Z level=info msg="Executing migration" id="Drop category_id index" 2024-04-13T07:30:38.703 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.507091988Z level=info msg="Executing migration" id="Add column tags to annotation table" 2024-04-13T07:30:38.703 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.549111004Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2024-04-13T07:30:38.704 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.574437621Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2024-04-13T07:30:38.704 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.616290526Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2024-04-13T07:30:38.704 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.658305389Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2024-04-13T07:30:38.704 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.708653037Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2024-04-13T07:30:38.770 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:38 smithi089 bash[31522]: cluster 2024-04-13T07:30:36.863411+0000 mgr.y (mgr.14373) 55 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:38.770 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:38 smithi089 bash[21353]: cluster 2024-04-13T07:30:36.863411+0000 mgr.y (mgr.14373) 55 : cluster 0 pgmap v36: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:38.972 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.733955833Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2024-04-13T07:30:38.972 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.759206151Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2024-04-13T07:30:38.972 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.78433627Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2024-04-13T07:30:38.972 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.809640728Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2024-04-13T07:30:38.973 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.834908632Z level=info msg="Executing migration" id="Add created time to annotation table" 2024-04-13T07:30:38.973 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.860061841Z level=info msg="Executing migration" id="Add updated time to annotation table" 2024-04-13T07:30:38.973 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.885424915Z level=info msg="Executing migration" id="Add index for created in annotation table" 2024-04-13T07:30:38.973 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.910621356Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2024-04-13T07:30:38.973 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.95260039Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2024-04-13T07:30:38.973 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:38 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:38.977762279Z level=info msg="Executing migration" id="Add epoch_end column" 2024-04-13T07:30:39.242 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.00304036Z level=info msg="Executing migration" id="Add index for epoch_end" 2024-04-13T07:30:39.242 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.04505005Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2024-04-13T07:30:39.242 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.070294048Z level=info msg="Executing migration" id="Move region to single row" 2024-04-13T07:30:39.242 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.095566903Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2024-04-13T07:30:39.242 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.120810211Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2024-04-13T07:30:39.242 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.146163705Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2024-04-13T07:30:39.242 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.179721078Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2024-04-13T07:30:39.242 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.196674531Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2024-04-13T07:30:39.243 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.213569645Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2024-04-13T07:30:39.496 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.247169231Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2024-04-13T07:30:39.496 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.280887049Z level=info msg="Executing migration" id="create test_data table" 2024-04-13T07:30:39.497 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.31466131Z level=info msg="Executing migration" id="create dashboard_version table v1" 2024-04-13T07:30:39.497 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.339832886Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2024-04-13T07:30:39.497 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.384104777Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2024-04-13T07:30:39.497 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.42595736Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2024-04-13T07:30:39.497 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.451212855Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2024-04-13T07:30:39.497 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.476510406Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2024-04-13T07:30:39.497 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.501762854Z level=info msg="Executing migration" id="create team table" 2024-04-13T07:30:39.756 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.543789277Z level=info msg="Executing migration" id="add index team.org_id" 2024-04-13T07:30:39.757 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.569155917Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2024-04-13T07:30:39.757 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.611101385Z level=info msg="Executing migration" id="create team member table" 2024-04-13T07:30:39.757 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.653056413Z level=info msg="Executing migration" id="add index team_member.org_id" 2024-04-13T07:30:39.757 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.695183335Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2024-04-13T07:30:39.778 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:39 smithi089 systemd[1]: Stopping Ceph alertmanager.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:30:39.778 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:39 smithi089 bash[56462]: ts=2024-04-13T07:30:39.623Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-13T07:30:40.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[23480]: cluster 2024-04-13T07:30:38.863975+0000 mgr.y (mgr.14373) 56 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:40.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[23480]: audit 2024-04-13T07:30:38.929734+0000 mon.a (mon.0) 752 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:40.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[23480]: audit 2024-04-13T07:30:38.930746+0000 mon.a (mon.0) 753 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:40.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.76219727Z level=info msg="Executing migration" id="add index team_member.team_id" 2024-04-13T07:30:40.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.787389795Z level=info msg="Executing migration" id="Add column email to team table" 2024-04-13T07:30:40.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.82955709Z level=info msg="Executing migration" id="Add column external to team_member table" 2024-04-13T07:30:40.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.863218342Z level=info msg="Executing migration" id="Add column permission to team_member table" 2024-04-13T07:30:40.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.913747835Z level=info msg="Executing migration" id="create dashboard acl table" 2024-04-13T07:30:40.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:39 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:39.964253834Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2024-04-13T07:30:40.017 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.006285602Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2024-04-13T07:30:40.050 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:40 smithi089 bash[58095]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-alertmanager-a 2024-04-13T07:30:40.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:39 smithi089 bash[21353]: cluster 2024-04-13T07:30:38.863975+0000 mgr.y (mgr.14373) 56 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:40.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:39 smithi089 bash[21353]: audit 2024-04-13T07:30:38.929734+0000 mon.a (mon.0) 752 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:40.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:39 smithi089 bash[21353]: audit 2024-04-13T07:30:38.930746+0000 mon.a (mon.0) 753 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:40.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:39 smithi089 bash[31522]: cluster 2024-04-13T07:30:38.863975+0000 mgr.y (mgr.14373) 56 : cluster 0 pgmap v37: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:40.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:39 smithi089 bash[31522]: audit 2024-04-13T07:30:38.929734+0000 mon.a (mon.0) 752 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:40.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:39 smithi089 bash[31522]: audit 2024-04-13T07:30:38.930746+0000 mon.a (mon.0) 753 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.04823954Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.081964055Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.098913568Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.115930823Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.141113857Z level=info msg="Executing migration" id="add index dashboard_permission" 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.191609814Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.216975005Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.25911284Z level=info msg="Executing migration" id="create tag table" 2024-04-13T07:30:40.296 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.301240916Z level=info msg="Executing migration" id="add index tag.key_value" 2024-04-13T07:30:40.548 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:40 smithi089 bash[58172]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-alertmanager-a 2024-04-13T07:30:40.548 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:40 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-13T07:30:40.549 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:40 smithi089 systemd[1]: Stopped Ceph alertmanager.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:30:40.549 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:40 smithi089 systemd[1]: Started Ceph alertmanager.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:30:40.565 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.318142461Z level=info msg="Executing migration" id="create login attempt table" 2024-04-13T07:30:40.565 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.335032734Z level=info msg="Executing migration" id="add index login_attempt.username" 2024-04-13T07:30:40.565 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.35198846Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2024-04-13T07:30:40.565 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.368905834Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2024-04-13T07:30:40.565 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.394152063Z level=info msg="Executing migration" id="create login_attempt v2" 2024-04-13T07:30:40.565 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.419425115Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2024-04-13T07:30:40.566 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.444646557Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2024-04-13T07:30:40.566 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.469931092Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2024-04-13T07:30:40.566 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.545458161Z level=info msg="Executing migration" id="create user auth table" 2024-04-13T07:30:40.566 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.570726089Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2024-04-13T07:30:40.961 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.612740986Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2024-04-13T07:30:40.961 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.654798044Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2024-04-13T07:30:40.961 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.671778199Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2024-04-13T07:30:40.961 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.688775033Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2024-04-13T07:30:40.961 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.705647866Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2024-04-13T07:30:40.961 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.722609279Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2024-04-13T07:30:40.961 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.806347339Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2024-04-13T07:30:40.961 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:40 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:40.966245874Z level=info msg="Executing migration" id="create server_lock table" 2024-04-13T07:30:41.265 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:41.024967168Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2024-04-13T07:30:41.265 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:41.125382895Z level=info msg="Executing migration" id="create user auth token table" 2024-04-13T07:30:41.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[21353]: audit 2024-04-13T07:30:40.245085+0000 mon.a (mon.0) 754 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:41.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[21353]: audit 2024-04-13T07:30:40.250875+0000 mon.a (mon.0) 755 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:41.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[21353]: cephadm 2024-04-13T07:30:40.274114+0000 mgr.y (mgr.14373) 57 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:30:41.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[21353]: cephadm 2024-04-13T07:30:40.509954+0000 mgr.y (mgr.14373) 58 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:30:41.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[31522]: audit 2024-04-13T07:30:40.245085+0000 mon.a (mon.0) 754 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:41.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[31522]: audit 2024-04-13T07:30:40.250875+0000 mon.a (mon.0) 755 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:41.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[31522]: cephadm 2024-04-13T07:30:40.274114+0000 mgr.y (mgr.14373) 57 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:30:41.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[31522]: cephadm 2024-04-13T07:30:40.509954+0000 mgr.y (mgr.14373) 58 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:30:41.300 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[58201]: ts=2024-04-13T07:30:41.095Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2024-04-13T07:30:41.300 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[58201]: ts=2024-04-13T07:30:41.095Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2024-04-13T07:30:41.300 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[58201]: ts=2024-04-13T07:30:41.097Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.89 port=9094 2024-04-13T07:30:41.300 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[58201]: ts=2024-04-13T07:30:41.098Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2024-04-13T07:30:41.300 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[58201]: ts=2024-04-13T07:30:41.128Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-13T07:30:41.300 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[58201]: ts=2024-04-13T07:30:41.128Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2024-04-13T07:30:41.300 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[58201]: ts=2024-04-13T07:30:41.132Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2024-04-13T07:30:41.300 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:41 smithi089 bash[58201]: ts=2024-04-13T07:30:41.132Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2024-04-13T07:30:41.516 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:41.33455737Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2024-04-13T07:30:41.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[23480]: audit 2024-04-13T07:30:40.245085+0000 mon.a (mon.0) 754 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:41.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[23480]: audit 2024-04-13T07:30:40.250875+0000 mon.a (mon.0) 755 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:41.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[23480]: cephadm 2024-04-13T07:30:40.274114+0000 mgr.y (mgr.14373) 57 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:30:41.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[23480]: cephadm 2024-04-13T07:30:40.509954+0000 mgr.y (mgr.14373) 58 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:30:42.016 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:41.580696788Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2024-04-13T07:30:42.016 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:41 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:41.711715872Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2024-04-13T07:30:42.294 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[23480]: cluster 2024-04-13T07:30:40.864758+0000 mgr.y (mgr.14373) 59 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:42.294 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.037719993Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2024-04-13T07:30:42.294 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.188271385Z level=info msg="Executing migration" id="create cache_data table" 2024-04-13T07:30:42.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:42 smithi089 bash[31522]: cluster 2024-04-13T07:30:40.864758+0000 mgr.y (mgr.14373) 59 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:42.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:42 smithi089 bash[21353]: cluster 2024-04-13T07:30:40.864758+0000 mgr.y (mgr.14373) 59 : cluster 0 pgmap v38: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:42.572 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.299337673Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2024-04-13T07:30:42.572 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.457505529Z level=info msg="Executing migration" id="create short_url table v1" 2024-04-13T07:30:42.846 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.577939602Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2024-04-13T07:30:42.846 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.640866205Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2024-04-13T07:30:42.846 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.699665523Z level=info msg="Executing migration" id="delete alert_definition table" 2024-04-13T07:30:42.846 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.801357008Z level=info msg="Executing migration" id="recreate alert_definition table" 2024-04-13T07:30:42.933 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:43.123 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.867676562Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2024-04-13T07:30:43.123 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:42 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:42.926795357Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2024-04-13T07:30:43.123 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:43 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:43.00252515Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2024-04-13T07:30:43.123 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:43 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:43.02766828Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2024-04-13T07:30:43.123 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:43 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:43.069544917Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2024-04-13T07:30:43.298 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:43 smithi089 bash[58201]: ts=2024-04-13T07:30:43.098Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000216926s 2024-04-13T07:30:43.413 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:43 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:43.128513234Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2024-04-13T07:30:43.414 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:43 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:43.296029737Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2024-04-13T07:30:43.414 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:43 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:43.419402375Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2024-04-13T07:30:44.004 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:43 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:43.689114393Z level=info msg="Executing migration" id="drop alert_definition table" 2024-04-13T07:30:44.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:44 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:44.009833701Z level=info msg="Executing migration" id="delete alert_definition_version table" 2024-04-13T07:30:44.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:44 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:44.157272538Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2024-04-13T07:30:44.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:44 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:44.216216013Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2024-04-13T07:30:44.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:44 smithi089 bash[31522]: cluster 2024-04-13T07:30:42.865310+0000 mgr.y (mgr.14373) 60 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:44 smithi089 bash[21353]: cluster 2024-04-13T07:30:42.865310+0000 mgr.y (mgr.14373) 60 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:44.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:44 smithi203 bash[23480]: cluster 2024-04-13T07:30:42.865310+0000 mgr.y (mgr.14373) 60 : cluster 0 pgmap v39: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:44.766 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:44 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:44.292793379Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2024-04-13T07:30:44.767 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:44 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:44.513875364Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2024-04-13T07:30:45.263 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:44 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:44.903554439Z level=info msg="Executing migration" id="drop alert_definition_version table" 2024-04-13T07:30:45.516 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:45 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:45.269389383Z level=info msg="Executing migration" id="create alert_instance table" 2024-04-13T07:30:45.516 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:45 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:45.455754641Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2024-04-13T07:30:45.566 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:30:45.567 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":64,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","created":"2024-04-13T07:19:16.936150+0000","modified":"2024-04-13T07:30:28.491216+0000","last_up_change":"2024-04-13T07:28:40.551327+0000","last_in_change":"2024-04-13T07:28:13.615937+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-13T07:24:24.430140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-13T07:30:20.226256+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9de691f5-25ed-472d-b6dd-dc3908fa85c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6803","nonce":3872923960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6805","nonce":3872923960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6809","nonce":3872923960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6807","nonce":3872923960}]},"public_addr":"172.21.15.89:6803/3872923960","cluster_addr":"172.21.15.89:6805/3872923960","heartbeat_back_addr":"172.21.15.89:6809/3872923960","heartbeat_front_addr":"172.21.15.89:6807/3872923960","state":["exists","up"]},{"osd":1,"uuid":"ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6811","nonce":302139657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6813","nonce":302139657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6817","nonce":302139657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6815","nonce":302139657}]},"public_addr":"172.21.15.89:6811/302139657","cluster_addr":"172.21.15.89:6813/302139657","heartbeat_back_addr":"172.21.15.89:6817/302139657","heartbeat_front_addr":"172.21.15.89:6815/302139657","state":["exists","up"]},{"osd":2,"uuid":"f86d6aea-97d3-4d75-99d0-6e5781d7a990","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6819","nonce":46732397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6821","nonce":46732397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6825","nonce":46732397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6823","nonce":46732397}]},"public_addr":"172.21.15.89:6819/46732397","cluster_addr":"172.21.15.89:6821/46732397","heartbeat_back_addr":"172.21.15.89:6825/46732397","heartbeat_front_addr":"172.21.15.89:6823/46732397","state":["exists","up"]},{"osd":3,"uuid":"37c263d3-d642-4cfa-99da-5a33e1bef16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6827","nonce":3306593060}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6829","nonce":3306593060}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6833","nonce":3306593060}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6831","nonce":3306593060}]},"public_addr":"172.21.15.89:6827/3306593060","cluster_addr":"172.21.15.89:6829/3306593060","heartbeat_back_addr":"172.21.15.89:6833/3306593060","heartbeat_front_addr":"172.21.15.89:6831/3306593060","state":["exists","up"]},{"osd":4,"uuid":"c1befaa2-c9f5-4023-9909-0577509ce44d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6801","nonce":3982510878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6803","nonce":3982510878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6807","nonce":3982510878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6805","nonce":3982510878}]},"public_addr":"172.21.15.203:6801/3982510878","cluster_addr":"172.21.15.203:6803/3982510878","heartbeat_back_addr":"172.21.15.203:6807/3982510878","heartbeat_front_addr":"172.21.15.203:6805/3982510878","state":["exists","up"]},{"osd":5,"uuid":"b45987d4-fc0f-45ad-9f0d-49240d898bfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6809","nonce":209623809}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6811","nonce":209623809}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6815","nonce":209623809}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6813","nonce":209623809}]},"public_addr":"172.21.15.203:6809/209623809","cluster_addr":"172.21.15.203:6811/209623809","heartbeat_back_addr":"172.21.15.203:6815/209623809","heartbeat_front_addr":"172.21.15.203:6813/209623809","state":["exists","up"]},{"osd":6,"uuid":"91bbf36b-799c-49df-8fbf-42a958699642","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6817","nonce":2138665907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6819","nonce":2138665907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6823","nonce":2138665907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6821","nonce":2138665907}]},"public_addr":"172.21.15.203:6817/2138665907","cluster_addr":"172.21.15.203:6819/2138665907","heartbeat_back_addr":"172.21.15.203:6823/2138665907","heartbeat_front_addr":"172.21.15.203:6821/2138665907","state":["exists","up"]},{"osd":7,"uuid":"aa9fb5c4-06e4-45ca-85f8-f2a96be15086","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6825","nonce":3848298771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6827","nonce":3848298771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6831","nonce":3848298771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6829","nonce":3848298771}]},"public_addr":"172.21.15.203:6825/3848298771","cluster_addr":"172.21.15.203:6827/3848298771","heartbeat_back_addr":"172.21.15.203:6831/3848298771","heartbeat_front_addr":"172.21.15.203:6829/3848298771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:22:41.252314+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:23:31.382195+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:24:20.791478+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:10.117632+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:59.130696+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:26:51.940707+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:27:44.966139+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:28:38.869265+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3115322972":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1139384373":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6800/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6801/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/28202064":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/2936084513":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6801/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/808783885":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/3574204772":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/868096906":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2680580613":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6801/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2483652022":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1215893273":"2024-04-14T07:20:19.796707+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-13T07:30:45.826 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:45 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:45.565420358Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2024-04-13T07:30:45.826 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:45 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:45.698785625Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2024-04-13T07:30:46.078 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:45 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:45.832378539Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2024-04-13T07:30:46.078 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:45 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:45.8745863Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2024-04-13T07:30:46.078 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:45 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:45.958544762Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2024-04-13T07:30:46.079 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.033810229Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2024-04-13T07:30:46.160 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-13T07:30:46.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-13T07:30:46.390 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[23480]: cluster 2024-04-13T07:30:44.865871+0000 mgr.y (mgr.14373) 61 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:46.390 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[23480]: audit 2024-04-13T07:30:45.570568+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.89:0/2678564422' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:46.390 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.084568561Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2024-04-13T07:30:46.394 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.135274775Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2024-04-13T07:30:46.394 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.202635493Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2024-04-13T07:30:46.394 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.227781639Z level=info msg="Executing migration" id="create alert_rule table" 2024-04-13T07:30:46.394 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.292303103Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2024-04-13T07:30:46.394 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.395735469Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2024-04-13T07:30:46.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:46 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:30:46] "GET /metrics HTTP/1.1" 200 28559 "" "Prometheus/2.43.0" 2024-04-13T07:30:46.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:46 smithi089 bash[21353]: cluster 2024-04-13T07:30:44.865871+0000 mgr.y (mgr.14373) 61 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:46.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:46 smithi089 bash[21353]: audit 2024-04-13T07:30:45.570568+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.89:0/2678564422' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:46.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:46 smithi089 bash[31522]: cluster 2024-04-13T07:30:44.865871+0000 mgr.y (mgr.14373) 61 : cluster 0 pgmap v40: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:46.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:46 smithi089 bash[31522]: audit 2024-04-13T07:30:45.570568+0000 mon.a (mon.0) 756 : audit 0 from='client.? 172.21.15.89:0/2678564422' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:46.692 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.531865938Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2024-04-13T07:30:46.962 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.698044747Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2024-04-13T07:30:47.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:46 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:46.964554065Z level=info msg="Executing migration" id="add column for to alert_rule" 2024-04-13T07:30:47.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.144012839Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2024-04-13T07:30:47.588 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 systemd[1]: Stopping Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:30:47.589 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.290343962Z level=info msg="Executing migration" id="add column labels to alert_rule" 2024-04-13T07:30:47.589 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.3745974Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2024-04-13T07:30:47.589 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.425664638Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2024-04-13T07:30:47.589 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.476166016Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2024-04-13T07:30:47.589 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.526847078Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2024-04-13T07:30:47.589 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.594293968Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2024-04-13T07:30:47.909 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.636644712Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2024-04-13T07:30:47.909 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.704044824Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2024-04-13T07:30:47.909 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.798242028Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2024-04-13T07:30:47.909 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:47.914333784Z level=info msg="Executing migration" id="create alert_rule_version table" 2024-04-13T07:30:47.909 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.617Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-13T07:30:47.909 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.617Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-13T07:30:47.909 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.617Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.617Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.617Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.617Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.617Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.617Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.618Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.618Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.618Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-13T07:30:47.910 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:47 smithi203 bash[48001]: ts=2024-04-13T07:30:47.618Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-13T07:30:48.246 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:48.026093014Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2024-04-13T07:30:48.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[23480]: cluster 2024-04-13T07:30:46.866516+0000 mgr.y (mgr.14373) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:48.516 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:48.251710654Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2024-04-13T07:30:48.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:48 smithi089 bash[31522]: cluster 2024-04-13T07:30:46.866516+0000 mgr.y (mgr.14373) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:48.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:48 smithi089 bash[21353]: cluster 2024-04-13T07:30:46.866516+0000 mgr.y (mgr.14373) 62 : cluster 0 pgmap v41: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:48.821 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:48.576662029Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2024-04-13T07:30:48.821 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:48.682956133Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2024-04-13T07:30:48.821 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:48.758861206Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2024-04-13T07:30:48.821 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[50369]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-prometheus-a 2024-04-13T07:30:48.821 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[50429]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-prometheus-a 2024-04-13T07:30:48.821 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:48 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-13T07:30:48.822 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:48 smithi203 systemd[1]: Stopped Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:30:48.822 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:48 smithi203 systemd[1]: Started Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:30:49.193 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:48.827017769Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2024-04-13T07:30:49.193 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:48 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:48.944248663Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:48 smithi089 bash[21606]: [13/Apr/2024:07:30:48] ENGINE Bus STOPPING 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:48 smithi089 bash[21606]: [13/Apr/2024:07:30:48] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:48 smithi089 bash[21606]: [13/Apr/2024:07:30:48] ENGINE Bus STOPPED 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:48 smithi089 bash[21606]: [13/Apr/2024:07:30:48] ENGINE Bus STARTING 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Serving on http://:::9283 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STARTED 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STOPPING 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-13T07:30:49.261 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STOPPED 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STARTING 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Serving on http://:::9283 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STARTED 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STOPPING 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STOPPED 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STARTING 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Serving on http://:::9283 2024-04-13T07:30:49.262 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21606]: [13/Apr/2024:07:30:49] ENGINE Bus STARTED 2024-04-13T07:30:49.516 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:49.198865448Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2024-04-13T07:30:49.516 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:49.416933444Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2024-04-13T07:30:49.986 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:49.722869524Z level=info msg="Executing migration" id=create_alert_configuration_table 2024-04-13T07:30:49.986 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:49.873785086Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2024-04-13T07:30:49.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.856520+0000 mon.a (mon.0) 757 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:49.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.864743+0000 mon.a (mon.0) 758 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: cluster 2024-04-13T07:30:48.866931+0000 mgr.y (mgr.14373) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.904402+0000 mon.a (mon.0) 759 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.904875+0000 mgr.y (mgr.14373) 64 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.906074+0000 mon.a (mon.0) 760 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.906407+0000 mgr.y (mgr.14373) 65 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.913318+0000 mon.a (mon.0) 761 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.929064+0000 mon.a (mon.0) 762 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.929607+0000 mgr.y (mgr.14373) 66 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.931080+0000 mon.a (mon.0) 763 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.931529+0000 mgr.y (mgr.14373) 67 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.939279+0000 mon.a (mon.0) 764 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.949764+0000 mon.a (mon.0) 765 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.950300+0000 mgr.y (mgr.14373) 68 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.952562+0000 mon.a (mon.0) 766 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-04-13T07:30:49.987 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.952946+0000 mgr.y (mgr.14373) 69 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-04-13T07:30:49.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.958275+0000 mon.a (mon.0) 767 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:49.988 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[23480]: audit 2024-04-13T07:30:48.973184+0000 mon.a (mon.0) 768 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:30:50.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:49 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:49.991645275Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2024-04-13T07:30:50.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.116931128Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2024-04-13T07:30:50.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.201064357Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.856520+0000 mon.a (mon.0) 757 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.864743+0000 mon.a (mon.0) 758 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: cluster 2024-04-13T07:30:48.866931+0000 mgr.y (mgr.14373) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.904402+0000 mon.a (mon.0) 759 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.904875+0000 mgr.y (mgr.14373) 64 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.906074+0000 mon.a (mon.0) 760 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.906407+0000 mgr.y (mgr.14373) 65 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.913318+0000 mon.a (mon.0) 761 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.929064+0000 mon.a (mon.0) 762 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.929607+0000 mgr.y (mgr.14373) 66 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.931080+0000 mon.a (mon.0) 763 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.931529+0000 mgr.y (mgr.14373) 67 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.939279+0000 mon.a (mon.0) 764 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.949764+0000 mon.a (mon.0) 765 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.950300+0000 mgr.y (mgr.14373) 68 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.952562+0000 mon.a (mon.0) 766 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.952946+0000 mgr.y (mgr.14373) 69 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.958275+0000 mon.a (mon.0) 767 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[21353]: audit 2024-04-13T07:30:48.973184+0000 mon.a (mon.0) 768 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.856520+0000 mon.a (mon.0) 757 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.864743+0000 mon.a (mon.0) 758 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: cluster 2024-04-13T07:30:48.866931+0000 mgr.y (mgr.14373) 63 : cluster 0 pgmap v42: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.904402+0000 mon.a (mon.0) 759 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.904875+0000 mgr.y (mgr.14373) 64 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.906074+0000 mon.a (mon.0) 760 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.906407+0000 mgr.y (mgr.14373) 65 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi089.front.sepia.ceph.com:9093"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.913318+0000 mon.a (mon.0) 761 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.929064+0000 mon.a (mon.0) 762 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.929607+0000 mgr.y (mgr.14373) 66 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.931080+0000 mon.a (mon.0) 763 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.931529+0000 mgr.y (mgr.14373) 67 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi203.front.sepia.ceph.com:9095"}]: dispatch 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.939279+0000 mon.a (mon.0) 764 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.949764+0000 mon.a (mon.0) 765 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-13T07:30:50.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.950300+0000 mgr.y (mgr.14373) 68 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2024-04-13T07:30:50.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.952562+0000 mon.a (mon.0) 766 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-04-13T07:30:50.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.952946+0000 mgr.y (mgr.14373) 69 : audit 0 from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi203.front.sepia.ceph.com:3000"}]: dispatch 2024-04-13T07:30:50.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.958275+0000 mon.a (mon.0) 767 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:50.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:49 smithi089 bash[31522]: audit 2024-04-13T07:30:48.973184+0000 mon.a (mon.0) 768 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:30:50.558 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.293285782Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2024-04-13T07:30:50.558 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.371864161Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2024-04-13T07:30:50.558 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.441935666Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2024-04-13T07:30:50.558 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.503098941Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2024-04-13T07:30:50.558 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.563219969Z level=info msg="Executing migration" id="create provenance_type table" 2024-04-13T07:30:50.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.654771714Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2024-04-13T07:30:50.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.68844959Z level=info msg="Executing migration" id="create alert_image table" 2024-04-13T07:30:50.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.73887143Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2024-04-13T07:30:50.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.772410638Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2024-04-13T07:30:50.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.806079386Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2024-04-13T07:30:50.851 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.856545451Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2024-04-13T07:30:50.851 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.576Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.576Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.576Z caller=main.go:570 level=info host_details="(Linux 5.15.0-102-generic #112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024 x86_64 smithi203 (none))" 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.576Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.576Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.578Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.578Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.579Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.579Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.582Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.582Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.373µs 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.582Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.584Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2024-04-13T07:30:50.852 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.584Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2024-04-13T07:30:50.853 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.584Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=28.308µs wal_replay_duration=2.078463ms wbl_replay_duration=97ns total_replay_duration=2.132302ms 2024-04-13T07:30:50.853 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.585Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-13T07:30:50.853 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.585Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-13T07:30:50.853 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.585Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-13T07:30:50.853 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.607Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=21.340567ms db_storage=999ns remote_storage=1.458µs web_handler=375ns query_engine=677ns scrape=260.288µs scrape_sd=90.537µs notify=10.908µs notify_sd=43.792µs rules=20.665849ms tracing=6.299µs 2024-04-13T07:30:50.853 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.607Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-13T07:30:50.853 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[50456]: ts=2024-04-13T07:30:50.607Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-13T07:30:50.887 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:51.120 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.890168436Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2024-04-13T07:30:51.120 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.890519038Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2024-04-13T07:30:51.120 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.940408844Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2024-04-13T07:30:51.120 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:50 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:50.965764747Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2024-04-13T07:30:51.121 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.007745955Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2024-04-13T07:30:51.121 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.008021058Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2024-04-13T07:30:51.121 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.008296006Z level=info msg="alerts found to migrate" alerts=0 2024-04-13T07:30:51.121 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.033076006Z level=info msg="Executing migration" id="create library_element table v1" 2024-04-13T07:30:51.121 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.058432554Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2024-04-13T07:30:51.121 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.100516105Z level=info msg="Executing migration" id="create library_element_connection table v1" 2024-04-13T07:30:51.121 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.125665378Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2024-04-13T07:30:51.298 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:30:51 smithi089 bash[58201]: ts=2024-04-13T07:30:51.101Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.002626761s 2024-04-13T07:30:51.414 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.168069204Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2024-04-13T07:30:51.414 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.218305014Z level=info msg="Executing migration" id="increase max description length to 2048" 2024-04-13T07:30:51.415 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.268670111Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2024-04-13T07:30:51.415 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.293979658Z level=info msg="Executing migration" id="create data_keys table" 2024-04-13T07:30:51.415 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.319321794Z level=info msg="Executing migration" id="create secrets table" 2024-04-13T07:30:51.415 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.36145909Z level=info msg="Executing migration" id="rename data_keys name column to id" 2024-04-13T07:30:51.415 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.420048498Z level=info msg="Executing migration" id="add name column into data_keys" 2024-04-13T07:30:51.666 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.445352156Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2024-04-13T07:30:51.666 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.487349978Z level=info msg="Executing migration" id="rename data_keys name column to label" 2024-04-13T07:30:51.666 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.546003532Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2024-04-13T07:30:51.666 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.588094551Z level=info msg="Executing migration" id="create kv_store table v1" 2024-04-13T07:30:51.667 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.630278747Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2024-04-13T07:30:51.667 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.672130563Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2024-04-13T07:30:51.917 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.697561467Z level=info msg="Executing migration" id="create permission table" 2024-04-13T07:30:51.917 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.722855203Z level=info msg="Executing migration" id="add unique index permission.role_id" 2024-04-13T07:30:51.917 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.748128935Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2024-04-13T07:30:51.917 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.773509473Z level=info msg="Executing migration" id="create role table" 2024-04-13T07:30:51.917 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.807065182Z level=info msg="Executing migration" id="add column display_name" 2024-04-13T07:30:51.917 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.857428062Z level=info msg="Executing migration" id="add column group_name" 2024-04-13T07:30:51.917 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.891031764Z level=info msg="Executing migration" id="add index role.org_id" 2024-04-13T07:30:51.985 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:51 smithi089 bash[21353]: cluster 2024-04-13T07:30:50.867511+0000 mgr.y (mgr.14373) 70 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:51.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:51 smithi089 bash[31522]: cluster 2024-04-13T07:30:50.867511+0000 mgr.y (mgr.14373) 70 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:52.213 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[23480]: cluster 2024-04-13T07:30:50.867511+0000 mgr.y (mgr.14373) 70 : cluster 0 pgmap v43: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:52.213 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.941644712Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2024-04-13T07:30:52.214 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:51 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:51.975214168Z level=info msg="Executing migration" id="add index role_org_id_uid" 2024-04-13T07:30:52.214 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.025553889Z level=info msg="Executing migration" id="create team role table" 2024-04-13T07:30:52.214 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.084611259Z level=info msg="Executing migration" id="add index team_role.org_id" 2024-04-13T07:30:52.214 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.13497145Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2024-04-13T07:30:52.214 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.168724444Z level=info msg="Executing migration" id="add index team_role.team_id" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.219060329Z level=info msg="Executing migration" id="create user role table" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.269445977Z level=info msg="Executing migration" id="add index user_role.org_id" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.28638184Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.303296443Z level=info msg="Executing migration" id="add index user_role.user_id" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.320218779Z level=info msg="Executing migration" id="create builtin role table" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.337243611Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.354232477Z level=info msg="Executing migration" id="add index builtin_role.name" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.371184833Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.413140675Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.455176174Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2024-04-13T07:30:52.475 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.480472812Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2024-04-13T07:30:52.727 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.505678023Z level=info msg="Executing migration" id="add unique index role.uid" 2024-04-13T07:30:52.727 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.531025844Z level=info msg="Executing migration" id="create seed assignment table" 2024-04-13T07:30:52.727 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.573079682Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2024-04-13T07:30:52.727 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.615126307Z level=info msg="Executing migration" id="add column hidden to role table" 2024-04-13T07:30:52.727 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.657070219Z level=info msg="Executing migration" id="create query_history table v1" 2024-04-13T07:30:52.727 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.68248791Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2024-04-13T07:30:52.727 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.707633938Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2024-04-13T07:30:52.997 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.732938151Z level=info msg="Executing migration" id="teams permissions migration" 2024-04-13T07:30:52.997 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.758199583Z level=info msg="Executing migration" id="dashboard permissions" 2024-04-13T07:30:52.997 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.783467693Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2024-04-13T07:30:52.997 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.808627214Z level=info msg="Executing migration" id="drop managed folder create actions" 2024-04-13T07:30:52.997 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.833830414Z level=info msg="Executing migration" id="alerting notification permissions" 2024-04-13T07:30:52.997 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.859093587Z level=info msg="Executing migration" id="create query_history_star table v1" 2024-04-13T07:30:52.997 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.884617627Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2024-04-13T07:30:52.998 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.910034639Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2024-04-13T07:30:52.998 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.952041932Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2024-04-13T07:30:52.998 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:52 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:52.977395558Z level=info msg="Executing migration" id="create correlation table v1" 2024-04-13T07:30:52.998 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.002705823Z level=info msg="Executing migration" id="add index correlations.uid" 2024-04-13T07:30:53.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.044683969Z level=info msg="Executing migration" id="add index correlations.source_uid" 2024-04-13T07:30:53.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.069959146Z level=info msg="Executing migration" id="add correlation config column" 2024-04-13T07:30:53.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.095166356Z level=info msg="Executing migration" id="create entity_events table" 2024-04-13T07:30:53.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.137210695Z level=info msg="Executing migration" id="create dashboard public config v1" 2024-04-13T07:30:53.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.179419574Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-13T07:30:53.267 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.179817969Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2024-04-13T07:30:53.267 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.229705851Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-13T07:30:53.267 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.230106039Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-13T07:30:53.537 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:30:53.537 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":64,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","created":"2024-04-13T07:19:16.936150+0000","modified":"2024-04-13T07:30:28.491216+0000","last_up_change":"2024-04-13T07:28:40.551327+0000","last_in_change":"2024-04-13T07:28:13.615937+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-13T07:24:24.430140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-13T07:30:20.226256+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9de691f5-25ed-472d-b6dd-dc3908fa85c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":57,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6803","nonce":3872923960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6805","nonce":3872923960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6809","nonce":3872923960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6807","nonce":3872923960}]},"public_addr":"172.21.15.89:6803/3872923960","cluster_addr":"172.21.15.89:6805/3872923960","heartbeat_back_addr":"172.21.15.89:6809/3872923960","heartbeat_front_addr":"172.21.15.89:6807/3872923960","state":["exists","up"]},{"osd":1,"uuid":"ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6811","nonce":302139657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6813","nonce":302139657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6817","nonce":302139657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6815","nonce":302139657}]},"public_addr":"172.21.15.89:6811/302139657","cluster_addr":"172.21.15.89:6813/302139657","heartbeat_back_addr":"172.21.15.89:6817/302139657","heartbeat_front_addr":"172.21.15.89:6815/302139657","state":["exists","up"]},{"osd":2,"uuid":"f86d6aea-97d3-4d75-99d0-6e5781d7a990","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6819","nonce":46732397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6821","nonce":46732397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6825","nonce":46732397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6823","nonce":46732397}]},"public_addr":"172.21.15.89:6819/46732397","cluster_addr":"172.21.15.89:6821/46732397","heartbeat_back_addr":"172.21.15.89:6825/46732397","heartbeat_front_addr":"172.21.15.89:6823/46732397","state":["exists","up"]},{"osd":3,"uuid":"37c263d3-d642-4cfa-99da-5a33e1bef16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6827","nonce":3306593060}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6829","nonce":3306593060}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6833","nonce":3306593060}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6831","nonce":3306593060}]},"public_addr":"172.21.15.89:6827/3306593060","cluster_addr":"172.21.15.89:6829/3306593060","heartbeat_back_addr":"172.21.15.89:6833/3306593060","heartbeat_front_addr":"172.21.15.89:6831/3306593060","state":["exists","up"]},{"osd":4,"uuid":"c1befaa2-c9f5-4023-9909-0577509ce44d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6801","nonce":3982510878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6803","nonce":3982510878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6807","nonce":3982510878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6805","nonce":3982510878}]},"public_addr":"172.21.15.203:6801/3982510878","cluster_addr":"172.21.15.203:6803/3982510878","heartbeat_back_addr":"172.21.15.203:6807/3982510878","heartbeat_front_addr":"172.21.15.203:6805/3982510878","state":["exists","up"]},{"osd":5,"uuid":"b45987d4-fc0f-45ad-9f0d-49240d898bfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6809","nonce":209623809}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6811","nonce":209623809}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6815","nonce":209623809}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6813","nonce":209623809}]},"public_addr":"172.21.15.203:6809/209623809","cluster_addr":"172.21.15.203:6811/209623809","heartbeat_back_addr":"172.21.15.203:6815/209623809","heartbeat_front_addr":"172.21.15.203:6813/209623809","state":["exists","up"]},{"osd":6,"uuid":"91bbf36b-799c-49df-8fbf-42a958699642","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6817","nonce":2138665907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6819","nonce":2138665907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6823","nonce":2138665907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6821","nonce":2138665907}]},"public_addr":"172.21.15.203:6817/2138665907","cluster_addr":"172.21.15.203:6819/2138665907","heartbeat_back_addr":"172.21.15.203:6823/2138665907","heartbeat_front_addr":"172.21.15.203:6821/2138665907","state":["exists","up"]},{"osd":7,"uuid":"aa9fb5c4-06e4-45ca-85f8-f2a96be15086","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6825","nonce":3848298771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6827","nonce":3848298771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6831","nonce":3848298771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6829","nonce":3848298771}]},"public_addr":"172.21.15.203:6825/3848298771","cluster_addr":"172.21.15.203:6827/3848298771","heartbeat_back_addr":"172.21.15.203:6831/3848298771","heartbeat_front_addr":"172.21.15.203:6829/3848298771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:22:41.252314+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:23:31.382195+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:24:20.791478+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:10.117632+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:59.130696+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:26:51.940707+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:27:44.966139+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:28:38.869265+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/3115322972":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1139384373":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6800/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6801/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/28202064":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/2936084513":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6801/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/808783885":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/3574204772":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/868096906":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2680580613":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6801/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2483652022":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1215893273":"2024-04-14T07:20:19.796707+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-13T07:30:53.568 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.279993593Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2024-04-13T07:30:53.568 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.330462754Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2024-04-13T07:30:53.568 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.38080495Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2024-04-13T07:30:53.568 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.414455952Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2024-04-13T07:30:53.568 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.448142687Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2024-04-13T07:30:53.568 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.498499376Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-13T07:30:53.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.573845255Z level=info msg="Executing migration" id="Drop public config table" 2024-04-13T07:30:53.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.624287122Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2024-04-13T07:30:53.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.674590685Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2024-04-13T07:30:53.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.724981116Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2024-04-13T07:30:53.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.750317958Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2024-04-13T07:30:53.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.775590324Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2024-04-13T07:30:53.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.809157418Z level=info msg="Executing migration" id="add annotations_enabled column" 2024-04-13T07:30:53.829 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.834430175Z level=info msg="Executing migration" id="add time_selection_enabled column" 2024-04-13T07:30:54.081 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[23480]: cluster 2024-04-13T07:30:52.868076+0000 mgr.y (mgr.14373) 71 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:54.081 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[23480]: audit 2024-04-13T07:30:53.542966+0000 mon.a (mon.0) 769 : audit 0 from='client.? 172.21.15.89:0/3911626544' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:54.081 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[23480]: audit 2024-04-13T07:30:53.561819+0000 mon.a (mon.0) 770 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:54.082 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[23480]: audit 2024-04-13T07:30:53.569602+0000 mon.a (mon.0) 771 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:54.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.859986825Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2024-04-13T07:30:54.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.885230715Z level=info msg="Executing migration" id="add share column" 2024-04-13T07:30:54.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.910480748Z level=info msg="Executing migration" id="create default alerting folders" 2024-04-13T07:30:54.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.952625778Z level=info msg="Executing migration" id="create file table" 2024-04-13T07:30:54.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:53 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:53.977934774Z level=info msg="Executing migration" id="file table idx: path natural pk" 2024-04-13T07:30:54.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.003227331Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2024-04-13T07:30:54.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.045245887Z level=info msg="Executing migration" id="create file_meta table" 2024-04-13T07:30:54.082 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.087070089Z level=info msg="Executing migration" id="file table idx: path key" 2024-04-13T07:30:54.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-13T07:30:54.108 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-13T07:30:54.109 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-13T07:30:54.109 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-13T07:30:54.109 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-13T07:30:54.110 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-13T07:30:54.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-13T07:30:54.111 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-13T07:30:54.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:53 smithi089 bash[21353]: cluster 2024-04-13T07:30:52.868076+0000 mgr.y (mgr.14373) 71 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:54.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:53 smithi089 bash[21353]: audit 2024-04-13T07:30:53.542966+0000 mon.a (mon.0) 769 : audit 0 from='client.? 172.21.15.89:0/3911626544' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:54.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:53 smithi089 bash[21353]: audit 2024-04-13T07:30:53.561819+0000 mon.a (mon.0) 770 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:54.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:53 smithi089 bash[21353]: audit 2024-04-13T07:30:53.569602+0000 mon.a (mon.0) 771 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:54.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:53 smithi089 bash[31522]: cluster 2024-04-13T07:30:52.868076+0000 mgr.y (mgr.14373) 71 : cluster 0 pgmap v44: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:54.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:53 smithi089 bash[31522]: audit 2024-04-13T07:30:53.542966+0000 mon.a (mon.0) 769 : audit 0 from='client.? 172.21.15.89:0/3911626544' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:30:54.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:53 smithi089 bash[31522]: audit 2024-04-13T07:30:53.561819+0000 mon.a (mon.0) 770 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:54.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:53 smithi089 bash[31522]: audit 2024-04-13T07:30:53.569602+0000 mon.a (mon.0) 771 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:54.350 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.112286562Z level=info msg="Executing migration" id="set path collation in file table" 2024-04-13T07:30:54.350 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.154389448Z level=info msg="Executing migration" id="managed permissions migration" 2024-04-13T07:30:54.350 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.196286765Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2024-04-13T07:30:54.351 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.221617621Z level=info msg="Executing migration" id="RBAC action name migrator" 2024-04-13T07:30:54.351 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.263602187Z level=info msg="Executing migration" id="Add UID column to playlist" 2024-04-13T07:30:54.351 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.2889864Z level=info msg="Executing migration" id="Update uid column values in playlist" 2024-04-13T07:30:54.351 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.330958693Z level=info msg="Executing migration" id="Add index for uid in playlist" 2024-04-13T07:30:54.351 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.356211778Z level=info msg="Executing migration" id="update group index for alert rules" 2024-04-13T07:30:54.606 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.398177278Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2024-04-13T07:30:54.606 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.423602688Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2024-04-13T07:30:54.607 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.448794545Z level=info msg="Executing migration" id="add action column to seed_assignment" 2024-04-13T07:30:54.607 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.474047386Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2024-04-13T07:30:54.607 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.516084349Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2024-04-13T07:30:54.607 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.533043285Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2024-04-13T07:30:54.607 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.591811087Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2024-04-13T07:30:54.863 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.608687875Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2024-04-13T07:30:54.863 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.625671231Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2024-04-13T07:30:54.864 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.676012504Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2024-04-13T07:30:54.864 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.709603645Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2024-04-13T07:30:54.864 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.743558054Z level=info msg="Executing migration" id="create folder table" 2024-04-13T07:30:54.864 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.768788151Z level=info msg="Executing migration" id="Add index for parent_uid" 2024-04-13T07:30:54.864 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.794082748Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2024-04-13T07:30:54.864 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.869451575Z level=info msg="Executing migration" id="Update folder title length" 2024-04-13T07:30:55.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[23480]: audit 2024-04-13T07:30:53.928514+0000 mon.a (mon.0) 772 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:55.266 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.911519209Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2024-04-13T07:30:55.267 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=migrator t=2024-04-13T07:30:54.953674517Z level=info msg="migrations completed" performed=485 skipped=0 duration=23.758748769s 2024-04-13T07:30:55.267 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:54 smithi203 bash[49777]: logger=sqlstore t=2024-04-13T07:30:54.954430878Z level=info msg="Created default organization" 2024-04-13T07:30:55.267 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:55 smithi203 bash[49777]: logger=plugin.loader t=2024-04-13T07:30:55.074196512Z level=info msg="Plugin registered" pluginID=input 2024-04-13T07:30:55.267 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:55 smithi203 bash[49777]: logger=plugin.loader t=2024-04-13T07:30:55.113495114Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2024-04-13T07:30:55.267 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:55 smithi203 bash[49777]: logger=plugin.loader t=2024-04-13T07:30:55.113533997Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2024-04-13T07:30:55.268 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:55 smithi203 bash[49777]: logger=secrets t=2024-04-13T07:30:55.113587586Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2024-04-13T07:30:55.268 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:55 smithi203 bash[49777]: logger=query_data t=2024-04-13T07:30:55.115625348Z level=info msg="Query Service initialization" 2024-04-13T07:30:55.268 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:55 smithi203 bash[49777]: logger=live.push_http t=2024-04-13T07:30:55.118880072Z level=info msg="Live Push Gateway initialization" 2024-04-13T07:30:55.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:54 smithi089 bash[31522]: audit 2024-04-13T07:30:53.928514+0000 mon.a (mon.0) 772 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:55.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:54 smithi089 bash[21353]: audit 2024-04-13T07:30:53.928514+0000 mon.a (mon.0) 772 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:30:56.427 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=infra.usagestats.collector t=2024-04-13T07:30:56.072746286Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2024-04-13T07:30:56.427 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=provisioning.datasources t=2024-04-13T07:30:56.128656905Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2024-04-13T07:30:56.427 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=provisioning.datasources t=2024-04-13T07:30:56.191035788Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2024-04-13T07:30:56.427 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=provisioning.alerting t=2024-04-13T07:30:56.265100792Z level=info msg="starting to provision alerting" 2024-04-13T07:30:56.427 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=provisioning.alerting t=2024-04-13T07:30:56.265156361Z level=info msg="finished to provision alerting" 2024-04-13T07:30:56.427 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=grafanaStorageLogger t=2024-04-13T07:30:56.265444918Z level=info msg="storage starting" 2024-04-13T07:30:56.427 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=ngalert.state.manager t=2024-04-13T07:30:56.265776044Z level=info msg="Warming state cache for startup" 2024-04-13T07:30:56.428 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=http.server t=2024-04-13T07:30:56.271393251Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2024-04-13T07:30:56.428 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=ngalert.state.manager t=2024-04-13T07:30:56.319694063Z level=info msg="State cache has been initialized" states=0 duration=53.91605ms 2024-04-13T07:30:56.428 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=ngalert.multiorg.alertmanager t=2024-04-13T07:30:56.31979884Z level=info msg="starting MultiOrg Alertmanager" 2024-04-13T07:30:56.428 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=ticker t=2024-04-13T07:30:56.319851041Z level=info msg=starting first_tick=2024-04-13T07:31:00Z 2024-04-13T07:30:56.428 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=sqlstore.transactions t=2024-04-13T07:30:56.432337631Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-13T07:30:56.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:30:56] "GET /metrics HTTP/1.1" 200 28559 "" "Prometheus/2.43.0" 2024-04-13T07:30:56.729 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=sqlstore.transactions t=2024-04-13T07:30:56.537637429Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-13T07:30:56.729 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=sqlstore.transactions t=2024-04-13T07:30:56.548988659Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2024-04-13T07:30:56.729 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=sqlstore.transactions t=2024-04-13T07:30:56.560036962Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=2 code="database is locked" 2024-04-13T07:30:56.729 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[49777]: logger=sqlstore.transactions t=2024-04-13T07:30:56.571118605Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=3 code="database is locked" 2024-04-13T07:30:57.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[23480]: cluster 2024-04-13T07:30:54.868628+0000 mgr.y (mgr.14373) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:57.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[23480]: audit 2024-04-13T07:30:55.736973+0000 mon.a (mon.0) 773 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:57.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[23480]: audit 2024-04-13T07:30:55.748266+0000 mon.a (mon.0) 774 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:57.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[23480]: audit 2024-04-13T07:30:56.274169+0000 mon.a (mon.0) 775 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:57.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[23480]: audit 2024-04-13T07:30:56.276174+0000 mon.a (mon.0) 776 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:57.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:56 smithi203 bash[23480]: audit 2024-04-13T07:30:56.286890+0000 mon.a (mon.0) 777 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[21353]: cluster 2024-04-13T07:30:54.868628+0000 mgr.y (mgr.14373) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[21353]: audit 2024-04-13T07:30:55.736973+0000 mon.a (mon.0) 773 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[21353]: audit 2024-04-13T07:30:55.748266+0000 mon.a (mon.0) 774 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[21353]: audit 2024-04-13T07:30:56.274169+0000 mon.a (mon.0) 775 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[21353]: audit 2024-04-13T07:30:56.276174+0000 mon.a (mon.0) 776 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[21353]: audit 2024-04-13T07:30:56.286890+0000 mon.a (mon.0) 777 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[31522]: cluster 2024-04-13T07:30:54.868628+0000 mgr.y (mgr.14373) 72 : cluster 0 pgmap v45: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[31522]: audit 2024-04-13T07:30:55.736973+0000 mon.a (mon.0) 773 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[31522]: audit 2024-04-13T07:30:55.748266+0000 mon.a (mon.0) 774 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[31522]: audit 2024-04-13T07:30:56.274169+0000 mon.a (mon.0) 775 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:30:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[31522]: audit 2024-04-13T07:30:56.276174+0000 mon.a (mon.0) 776 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:30:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:56 smithi089 bash[31522]: audit 2024-04-13T07:30:56.286890+0000 mon.a (mon.0) 777 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:30:58.930 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:58.930 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:58.930 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:58.931 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:58.931 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:58.932 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:58.933 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:58.933 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:30:59.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:30:58 smithi203 bash[23480]: cluster 2024-04-13T07:30:56.869303+0000 mgr.y (mgr.14373) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:59.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:30:58 smithi089 bash[21353]: cluster 2024-04-13T07:30:56.869303+0000 mgr.y (mgr.14373) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:30:59.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:30:58 smithi089 bash[31522]: cluster 2024-04-13T07:30:56.869303+0000 mgr.y (mgr.14373) 73 : cluster 0 pgmap v46: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:01.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:00 smithi203 bash[23480]: cluster 2024-04-13T07:30:58.869877+0000 mgr.y (mgr.14373) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:01.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:00 smithi089 bash[31522]: cluster 2024-04-13T07:30:58.869877+0000 mgr.y (mgr.14373) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:01.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:00 smithi089 bash[21353]: cluster 2024-04-13T07:30:58.869877+0000 mgr.y (mgr.14373) 74 : cluster 0 pgmap v47: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:03.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:02 smithi203 bash[23480]: cluster 2024-04-13T07:31:00.870539+0000 mgr.y (mgr.14373) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:03.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:02 smithi089 bash[21353]: cluster 2024-04-13T07:31:00.870539+0000 mgr.y (mgr.14373) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:03.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:02 smithi089 bash[31522]: cluster 2024-04-13T07:31:00.870539+0000 mgr.y (mgr.14373) 75 : cluster 0 pgmap v48: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:05.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:04 smithi203 bash[23480]: cluster 2024-04-13T07:31:02.871144+0000 mgr.y (mgr.14373) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:05.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:04 smithi089 bash[21353]: cluster 2024-04-13T07:31:02.871144+0000 mgr.y (mgr.14373) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:05.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:04 smithi089 bash[31522]: cluster 2024-04-13T07:31:02.871144+0000 mgr.y (mgr.14373) 76 : cluster 0 pgmap v49: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:06.466 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:31:06 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:31:06] "GET /metrics HTTP/1.1" 200 28559 "" "Prometheus/2.43.0" 2024-04-13T07:31:07.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:06 smithi203 bash[23480]: cluster 2024-04-13T07:31:04.871702+0000 mgr.y (mgr.14373) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:07.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:06 smithi089 bash[21353]: cluster 2024-04-13T07:31:04.871702+0000 mgr.y (mgr.14373) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:07.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:06 smithi089 bash[31522]: cluster 2024-04-13T07:31:04.871702+0000 mgr.y (mgr.14373) 77 : cluster 0 pgmap v50: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:08.596 INFO:teuthology.orchestra.run.smithi089.stdout:77309411409 2024-04-13T07:31:08.597 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-13T07:31:09.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:08 smithi203 bash[23480]: cluster 2024-04-13T07:31:06.872363+0000 mgr.y (mgr.14373) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:09.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:08 smithi089 bash[21353]: cluster 2024-04-13T07:31:06.872363+0000 mgr.y (mgr.14373) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:09.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:08 smithi089 bash[31522]: cluster 2024-04-13T07:31:06.872363+0000 mgr.y (mgr.14373) 78 : cluster 0 pgmap v51: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:09.053 INFO:teuthology.orchestra.run.smithi089.stdout:201863462953 2024-04-13T07:31:09.053 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-13T07:31:10.009 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:09 smithi089 bash[21353]: audit 2024-04-13T07:31:08.929004+0000 mon.a (mon.0) 778 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:10.010 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:09 smithi089 bash[31522]: audit 2024-04-13T07:31:08.929004+0000 mon.a (mon.0) 778 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:10.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:09 smithi203 bash[23480]: audit 2024-04-13T07:31:08.929004+0000 mon.a (mon.0) 778 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:10.881 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:10 smithi089 bash[31522]: cluster 2024-04-13T07:31:08.872936+0000 mgr.y (mgr.14373) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:10.881 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:10 smithi089 bash[21353]: cluster 2024-04-13T07:31:08.872936+0000 mgr.y (mgr.14373) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:11.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:10 smithi203 bash[23480]: cluster 2024-04-13T07:31:08.872936+0000 mgr.y (mgr.14373) 79 : cluster 0 pgmap v52: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:11.696 INFO:teuthology.orchestra.run.smithi089.stdout:34359738472 2024-04-13T07:31:11.697 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-13T07:31:11.698 INFO:teuthology.orchestra.run.smithi089.stdout:236223201312 2024-04-13T07:31:11.698 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-13T07:31:11.700 INFO:teuthology.orchestra.run.smithi089.stdout:55834574941 2024-04-13T07:31:11.701 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-13T07:31:11.759 INFO:teuthology.orchestra.run.smithi089.stdout:167503724597 2024-04-13T07:31:11.759 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-13T07:31:12.460 INFO:teuthology.orchestra.run.smithi089.stdout:133143986239 2024-04-13T07:31:12.461 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-13T07:31:12.473 INFO:teuthology.orchestra.run.smithi089.stdout:111669149769 2024-04-13T07:31:12.473 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-13T07:31:13.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:12 smithi089 bash[31522]: cluster 2024-04-13T07:31:10.873615+0000 mgr.y (mgr.14373) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:13.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:12 smithi089 bash[21353]: cluster 2024-04-13T07:31:10.873615+0000 mgr.y (mgr.14373) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:13.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:12 smithi203 bash[23480]: cluster 2024-04-13T07:31:10.873615+0000 mgr.y (mgr.14373) 80 : cluster 0 pgmap v53: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:14.996 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:14.996 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:15.029 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:14 smithi089 bash[21353]: cluster 2024-04-13T07:31:12.874191+0000 mgr.y (mgr.14373) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:15.029 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:14 smithi089 bash[31522]: cluster 2024-04-13T07:31:12.874191+0000 mgr.y (mgr.14373) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:15.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:14 smithi203 bash[23480]: cluster 2024-04-13T07:31:12.874191+0000 mgr.y (mgr.14373) 81 : cluster 0 pgmap v54: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:16.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:31:16 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:31:16] "GET /metrics HTTP/1.1" 200 28554 "" "Prometheus/2.43.0" 2024-04-13T07:31:17.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:16 smithi089 bash[31522]: cluster 2024-04-13T07:31:14.874779+0000 mgr.y (mgr.14373) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:17.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:16 smithi089 bash[21353]: cluster 2024-04-13T07:31:14.874779+0000 mgr.y (mgr.14373) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:17.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:16 smithi203 bash[23480]: cluster 2024-04-13T07:31:14.874779+0000 mgr.y (mgr.14373) 82 : cluster 0 pgmap v55: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:17.550 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:17.551 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:17.551 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:17.552 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:17.552 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:17.552 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:18.932 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:18 smithi089 bash[31522]: cluster 2024-04-13T07:31:16.875478+0000 mgr.y (mgr.14373) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:18.932 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:18 smithi089 bash[21353]: cluster 2024-04-13T07:31:16.875478+0000 mgr.y (mgr.14373) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:19.064 INFO:teuthology.orchestra.run.smithi089.stdout:201863462955 2024-04-13T07:31:19.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:18 smithi203 bash[23480]: cluster 2024-04-13T07:31:16.875478+0000 mgr.y (mgr.14373) 83 : cluster 0 pgmap v56: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:19.507 INFO:teuthology.orchestra.run.smithi089.stdout:77309411412 2024-04-13T07:31:20.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:19 smithi089 bash[21353]: audit 2024-04-13T07:31:19.070737+0000 mon.a (mon.0) 779 : audit 0 from='client.? 172.21.15.89:0/1489506695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:31:20.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:19 smithi089 bash[21353]: audit 2024-04-13T07:31:19.514119+0000 mon.a (mon.0) 780 : audit 0 from='client.? 172.21.15.89:0/261081314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:31:20.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:19 smithi089 bash[31522]: audit 2024-04-13T07:31:19.070737+0000 mon.a (mon.0) 779 : audit 0 from='client.? 172.21.15.89:0/1489506695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:31:20.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:19 smithi089 bash[31522]: audit 2024-04-13T07:31:19.514119+0000 mon.a (mon.0) 780 : audit 0 from='client.? 172.21.15.89:0/261081314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:31:20.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:19 smithi203 bash[23480]: audit 2024-04-13T07:31:19.070737+0000 mon.a (mon.0) 779 : audit 0 from='client.? 172.21.15.89:0/1489506695' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:31:20.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:19 smithi203 bash[23480]: audit 2024-04-13T07:31:19.514119+0000 mon.a (mon.0) 780 : audit 0 from='client.? 172.21.15.89:0/261081314' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:31:21.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:20 smithi089 bash[31522]: cluster 2024-04-13T07:31:18.875807+0000 mgr.y (mgr.14373) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:21.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:20 smithi089 bash[21353]: cluster 2024-04-13T07:31:18.875807+0000 mgr.y (mgr.14373) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:21.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:20 smithi203 bash[23480]: cluster 2024-04-13T07:31:18.875807+0000 mgr.y (mgr.14373) 84 : cluster 0 pgmap v57: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:22.593 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462953 got 201863462955 for osd.6 2024-04-13T07:31:22.594 DEBUG:teuthology.parallel:result is None 2024-04-13T07:31:22.627 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411409 got 77309411412 for osd.2 2024-04-13T07:31:22.627 DEBUG:teuthology.parallel:result is None 2024-04-13T07:31:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:22 smithi089 bash[21353]: cluster 2024-04-13T07:31:20.876391+0000 mgr.y (mgr.14373) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:23.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:22 smithi089 bash[31522]: cluster 2024-04-13T07:31:20.876391+0000 mgr.y (mgr.14373) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:23.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:22 smithi203 bash[23480]: cluster 2024-04-13T07:31:20.876391+0000 mgr.y (mgr.14373) 85 : cluster 0 pgmap v58: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:23.670 INFO:teuthology.orchestra.run.smithi089.stdout:133143986241 2024-04-13T07:31:24.124 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:23 smithi089 bash[31522]: audit 2024-04-13T07:31:23.676852+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.89:0/2720409193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:31:24.124 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:23 smithi089 bash[21353]: audit 2024-04-13T07:31:23.676852+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.89:0/2720409193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:31:24.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:23 smithi203 bash[23480]: audit 2024-04-13T07:31:23.676852+0000 mon.c (mon.2) 17 : audit 0 from='client.? 172.21.15.89:0/2720409193' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:31:25.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:24 smithi089 bash[31522]: cluster 2024-04-13T07:31:22.876965+0000 mgr.y (mgr.14373) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:25.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:24 smithi089 bash[31522]: audit 2024-04-13T07:31:23.929716+0000 mon.a (mon.0) 781 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:25.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:24 smithi089 bash[21353]: cluster 2024-04-13T07:31:22.876965+0000 mgr.y (mgr.14373) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:25.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:24 smithi089 bash[21353]: audit 2024-04-13T07:31:23.929716+0000 mon.a (mon.0) 781 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:25.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:24 smithi203 bash[23480]: cluster 2024-04-13T07:31:22.876965+0000 mgr.y (mgr.14373) 86 : cluster 0 pgmap v59: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:25.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:24 smithi203 bash[23480]: audit 2024-04-13T07:31:23.929716+0000 mon.a (mon.0) 781 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:26.109 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:25 smithi089 bash[21353]: cluster 2024-04-13T07:31:24.877499+0000 mgr.y (mgr.14373) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:26.110 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:25 smithi089 bash[31522]: cluster 2024-04-13T07:31:24.877499+0000 mgr.y (mgr.14373) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:26.228 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986239 got 133143986241 for osd.4 2024-04-13T07:31:26.228 DEBUG:teuthology.parallel:result is None 2024-04-13T07:31:26.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:25 smithi203 bash[23480]: cluster 2024-04-13T07:31:24.877499+0000 mgr.y (mgr.14373) 87 : cluster 0 pgmap v60: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:26.490 INFO:teuthology.orchestra.run.smithi089.stdout:236223201314 2024-04-13T07:31:26.504 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:31:26 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:31:26] "GET /metrics HTTP/1.1" 200 28549 "" "Prometheus/2.43.0" 2024-04-13T07:31:27.138 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:26 smithi089 bash[21353]: audit 2024-04-13T07:31:26.496875+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.89:0/1614234299' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:31:27.139 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:26 smithi089 bash[31522]: audit 2024-04-13T07:31:26.496875+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.89:0/1614234299' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:31:27.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:26 smithi203 bash[23480]: audit 2024-04-13T07:31:26.496875+0000 mon.c (mon.2) 18 : audit 0 from='client.? 172.21.15.89:0/1614234299' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:31:27.512 INFO:teuthology.orchestra.run.smithi089.stdout:111669149771 2024-04-13T07:31:27.735 INFO:teuthology.orchestra.run.smithi089.stdout:167503724599 2024-04-13T07:31:28.035 INFO:teuthology.orchestra.run.smithi089.stdout:55834574944 2024-04-13T07:31:28.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:27 smithi089 bash[21353]: cluster 2024-04-13T07:31:26.878171+0000 mgr.y (mgr.14373) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:28.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:27 smithi089 bash[21353]: audit 2024-04-13T07:31:27.518834+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.89:0/521797984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:31:28.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:27 smithi089 bash[21353]: audit 2024-04-13T07:31:27.742191+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.89:0/369898158' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:31:28.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:27 smithi089 bash[31522]: cluster 2024-04-13T07:31:26.878171+0000 mgr.y (mgr.14373) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:28.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:27 smithi089 bash[31522]: audit 2024-04-13T07:31:27.518834+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.89:0/521797984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:31:28.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:27 smithi089 bash[31522]: audit 2024-04-13T07:31:27.742191+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.89:0/369898158' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:31:28.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:27 smithi203 bash[23480]: cluster 2024-04-13T07:31:26.878171+0000 mgr.y (mgr.14373) 88 : cluster 0 pgmap v61: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:28.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:27 smithi203 bash[23480]: audit 2024-04-13T07:31:27.518834+0000 mon.c (mon.2) 19 : audit 0 from='client.? 172.21.15.89:0/521797984' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:31:28.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:27 smithi203 bash[23480]: audit 2024-04-13T07:31:27.742191+0000 mon.c (mon.2) 20 : audit 0 from='client.? 172.21.15.89:0/369898158' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:31:28.484 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201312 got 236223201314 for osd.7 2024-04-13T07:31:28.484 DEBUG:teuthology.parallel:result is None 2024-04-13T07:31:28.817 INFO:teuthology.orchestra.run.smithi089.stdout:34359738475 2024-04-13T07:31:29.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:28 smithi203 bash[23480]: audit 2024-04-13T07:31:28.042198+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.89:0/3963933453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:31:29.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:28 smithi203 bash[23480]: audit 2024-04-13T07:31:28.823612+0000 mon.a (mon.0) 782 : audit 0 from='client.? 172.21.15.89:0/4186655330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:31:29.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:28 smithi089 bash[21353]: audit 2024-04-13T07:31:28.042198+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.89:0/3963933453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:31:29.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:28 smithi089 bash[21353]: audit 2024-04-13T07:31:28.823612+0000 mon.a (mon.0) 782 : audit 0 from='client.? 172.21.15.89:0/4186655330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:31:29.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:28 smithi089 bash[31522]: audit 2024-04-13T07:31:28.042198+0000 mon.c (mon.2) 21 : audit 0 from='client.? 172.21.15.89:0/3963933453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:31:29.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:28 smithi089 bash[31522]: audit 2024-04-13T07:31:28.823612+0000 mon.a (mon.0) 782 : audit 0 from='client.? 172.21.15.89:0/4186655330' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:31:29.538 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149769 got 111669149771 for osd.3 2024-04-13T07:31:29.538 DEBUG:teuthology.parallel:result is None 2024-04-13T07:31:29.758 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724597 got 167503724599 for osd.5 2024-04-13T07:31:29.758 DEBUG:teuthology.parallel:result is None 2024-04-13T07:31:29.949 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:29 smithi089 bash[21353]: cluster 2024-04-13T07:31:28.878711+0000 mgr.y (mgr.14373) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:29.950 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:29 smithi089 bash[31522]: cluster 2024-04-13T07:31:28.878711+0000 mgr.y (mgr.14373) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:29.950 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574941 got 55834574944 for osd.1 2024-04-13T07:31:29.951 DEBUG:teuthology.parallel:result is None 2024-04-13T07:31:30.192 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738472 got 34359738475 for osd.0 2024-04-13T07:31:30.192 DEBUG:teuthology.parallel:result is None 2024-04-13T07:31:30.193 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-13T07:31:30.193 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-13T07:31:30.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:29 smithi203 bash[23480]: cluster 2024-04-13T07:31:28.878711+0000 mgr.y (mgr.14373) 89 : cluster 0 pgmap v62: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:32.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:31 smithi203 bash[23480]: cluster 2024-04-13T07:31:30.879329+0000 mgr.y (mgr.14373) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:32.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:31 smithi089 bash[31522]: cluster 2024-04-13T07:31:30.879329+0000 mgr.y (mgr.14373) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:32.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:31 smithi089 bash[21353]: cluster 2024-04-13T07:31:30.879329+0000 mgr.y (mgr.14373) 90 : cluster 0 pgmap v63: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:34.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:33 smithi203 bash[23480]: cluster 2024-04-13T07:31:32.879834+0000 mgr.y (mgr.14373) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:34.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:33 smithi089 bash[31522]: cluster 2024-04-13T07:31:32.879834+0000 mgr.y (mgr.14373) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:34.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:33 smithi089 bash[21353]: cluster 2024-04-13T07:31:32.879834+0000 mgr.y (mgr.14373) 91 : cluster 0 pgmap v64: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:34.923 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:36.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:35 smithi203 bash[23480]: cluster 2024-04-13T07:31:34.880409+0000 mgr.y (mgr.14373) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:36.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:35 smithi089 bash[21353]: cluster 2024-04-13T07:31:34.880409+0000 mgr.y (mgr.14373) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:36.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:31:36 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:31:36] "GET /metrics HTTP/1.1" 200 28549 "" "Prometheus/2.43.0" 2024-04-13T07:31:36.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:35 smithi089 bash[31522]: cluster 2024-04-13T07:31:34.880409+0000 mgr.y (mgr.14373) 92 : cluster 0 pgmap v65: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:37.432 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:31:37.434 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-04-13T07:31:37.966 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":66,"stamp":"2024-04-13T07:31:36.880694+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":235200,"kb_used_data":6752,"kb_used_omap":12,"kb_used_meta":228339,"kb_avail":749726016,"statfs":{"total":767960285184,"available":767719440384,"internally_reserved":0,"allocated":6914048,"data_stored":3904681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":233819705},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003475"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.882284+0000","last_change":"2024-04-13T07:30:21.470196+0000","last_active":"2024-04-13T07:30:24.882284+0000","last_peered":"2024-04-13T07:30:24.882284+0000","last_clean":"2024-04-13T07:30:24.882284+0000","last_became_active":"2024-04-13T07:30:21.469340+0000","last_became_peered":"2024-04-13T07:30:21.469340+0000","last_unstale":"2024-04-13T07:30:24.882284+0000","last_undegraded":"2024-04-13T07:30:24.882284+0000","last_fullsized":"2024-04-13T07:30:24.882284+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:32:48.529436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.876804+0000","last_change":"2024-04-13T07:30:22.257370+0000","last_active":"2024-04-13T07:30:24.876804+0000","last_peered":"2024-04-13T07:30:24.876804+0000","last_clean":"2024-04-13T07:30:24.876804+0000","last_became_active":"2024-04-13T07:30:22.256904+0000","last_became_peered":"2024-04-13T07:30:22.256904+0000","last_unstale":"2024-04-13T07:30:24.876804+0000","last_undegraded":"2024-04-13T07:30:24.876804+0000","last_fullsized":"2024-04-13T07:30:24.876804+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:09:42.362257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.883082+0000","last_change":"2024-04-13T07:30:22.258054+0000","last_active":"2024-04-13T07:30:24.883082+0000","last_peered":"2024-04-13T07:30:24.883082+0000","last_clean":"2024-04-13T07:30:24.883082+0000","last_became_active":"2024-04-13T07:30:22.256961+0000","last_became_peered":"2024-04-13T07:30:22.256961+0000","last_unstale":"2024-04-13T07:30:24.883082+0000","last_undegraded":"2024-04-13T07:30:24.883082+0000","last_fullsized":"2024-04-13T07:30:24.883082+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:13:36.989961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.882984+0000","last_change":"2024-04-13T07:30:21.472711+0000","last_active":"2024-04-13T07:30:24.882984+0000","last_peered":"2024-04-13T07:30:24.882984+0000","last_clean":"2024-04-13T07:30:24.882984+0000","last_became_active":"2024-04-13T07:30:21.471979+0000","last_became_peered":"2024-04-13T07:30:21.471979+0000","last_unstale":"2024-04-13T07:30:24.882984+0000","last_undegraded":"2024-04-13T07:30:24.882984+0000","last_fullsized":"2024-04-13T07:30:24.882984+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:27:07.929194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":260,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-04-13T07:30:29.895446+0000","last_change":"2024-04-13T07:28:43.834029+0000","last_active":"2024-04-13T07:30:29.895446+0000","last_peered":"2024-04-13T07:30:29.895446+0000","last_clean":"2024-04-13T07:30:29.895446+0000","last_became_active":"2024-04-13T07:28:43.833192+0000","last_became_peered":"2024-04-13T07:28:43.833192+0000","last_unstale":"2024-04-13T07:30:29.895446+0000","last_undegraded":"2024-04-13T07:30:29.895446+0000","last_fullsized":"2024-04-13T07:30:29.895446+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_clean_scrub_stamp":"2024-04-13T07:24:24.657602+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:32:17.256871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.881728+0000","last_change":"2024-04-13T07:30:21.467433+0000","last_active":"2024-04-13T07:30:24.881728+0000","last_peered":"2024-04-13T07:30:24.881728+0000","last_clean":"2024-04-13T07:30:24.881728+0000","last_became_active":"2024-04-13T07:30:21.466602+0000","last_became_peered":"2024-04-13T07:30:21.466602+0000","last_unstale":"2024-04-13T07:30:24.881728+0000","last_undegraded":"2024-04-13T07:30:24.881728+0000","last_fullsized":"2024-04-13T07:30:24.881728+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:29:15.004301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.880732+0000","last_change":"2024-04-13T07:30:21.471551+0000","last_active":"2024-04-13T07:30:24.880732+0000","last_peered":"2024-04-13T07:30:24.880732+0000","last_clean":"2024-04-13T07:30:24.880732+0000","last_became_active":"2024-04-13T07:30:21.470542+0000","last_became_peered":"2024-04-13T07:30:21.470542+0000","last_unstale":"2024-04-13T07:30:24.880732+0000","last_undegraded":"2024-04-13T07:30:24.880732+0000","last_fullsized":"2024-04-13T07:30:24.880732+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:54:43.387491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.881691+0000","last_change":"2024-04-13T07:30:21.469386+0000","last_active":"2024-04-13T07:30:24.881691+0000","last_peered":"2024-04-13T07:30:24.881691+0000","last_clean":"2024-04-13T07:30:24.881691+0000","last_became_active":"2024-04-13T07:30:21.468314+0000","last_became_peered":"2024-04-13T07:30:21.468314+0000","last_unstale":"2024-04-13T07:30:24.881691+0000","last_undegraded":"2024-04-13T07:30:24.881691+0000","last_fullsized":"2024-04-13T07:30:24.881691+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:51:04.281850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.876767+0000","last_change":"2024-04-13T07:30:21.471173+0000","last_active":"2024-04-13T07:30:24.876767+0000","last_peered":"2024-04-13T07:30:24.876767+0000","last_clean":"2024-04-13T07:30:24.876767+0000","last_became_active":"2024-04-13T07:30:21.470193+0000","last_became_peered":"2024-04-13T07:30:21.470193+0000","last_unstale":"2024-04-13T07:30:24.876767+0000","last_undegraded":"2024-04-13T07:30:24.876767+0000","last_fullsized":"2024-04-13T07:30:24.876767+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:13:17.367790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201317,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32324,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712828,"statfs":{"total":95995035648,"available":95961935872,"internally_reserved":0,"allocated":1236992,"data_stored":857182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.86399999999999999,"15min":0.871},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.2250000000000001,"5min":1.3,"15min":1.3},"last":0.86599999999999999},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.92300000000000004,"15min":0.93300000000000005},"min":{"1min":0.59899999999999998,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.2030000000000001,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.64700000000000002}]},{"osd":1,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.88600000000000001,"15min":0.88},"min":{"1min":0.59199999999999997,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.75700000000000001},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"min":{"1min":0.58099999999999996,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.282,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":1.0780000000000001}]},{"osd":2,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.92000000000000004,"15min":0.92100000000000004},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.272,"5min":1.272,"15min":1.272},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.01,"15min":1.016},"min":{"1min":0.66500000000000004,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.2529999999999999,"5min":1.367,"15min":1.367},"last":1.0489999999999999}]},{"osd":3,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.94199999999999995,"15min":0.94199999999999995},"min":{"1min":0.60499999999999998,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.401,"5min":1.401,"15min":1.401},"last":0.79300000000000004},{"interface":"front","average":{"1min":1.016,"5min":1.036,"15min":1.04},"min":{"1min":0.61299999999999999,"5min":0.501,"15min":0.501},"max":{"1min":1.355,"5min":1.355,"15min":1.355},"last":1.153}]},{"osd":4,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.87,"15min":0.871},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":1.202},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.92000000000000004,"15min":0.92400000000000004},"min":{"1min":0.47399999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2989999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.129}]},{"osd":5,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.98599999999999999,"15min":0.99199999999999999},"min":{"1min":0.49099999999999999,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.3420000000000001,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":0.98099999999999998},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.03,"15min":1.04},"min":{"1min":0.498,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.262,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.361}]},{"osd":6,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.0900000000000001,"15min":1.0960000000000001},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3340000000000001,"5min":1.536,"15min":1.536},"last":1.28},{"interface":"front","average":{"1min":1.008,"5min":1.0840000000000001,"15min":1.0960000000000001},"min":{"1min":0.56799999999999995,"5min":0.434,"15min":0.434},"max":{"1min":1.3640000000000001,"5min":1.633,"15min":1.633},"last":1.014}]}]},{"osd":6,"up_from":47,"seq":201863462959,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1228800,"data_stored":856578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.86799999999999999,"15min":0.84799999999999998},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":0.85199999999999998},{"interface":"front","average":{"1min":0.875,"5min":0.88400000000000001,"15min":0.88},"min":{"1min":0.55300000000000005,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.234,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.113}]},{"osd":1,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.92100000000000004,"15min":0.92200000000000004},"min":{"1min":0.56899999999999995,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.327,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.93100000000000005,"15min":0.93300000000000005},"min":{"1min":0.60599999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.4470000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":1.0609999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.91700000000000004,"15min":0.90000000000000002},"min":{"1min":0.57299999999999995,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.1539999999999999},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.98899999999999999,"15min":1.0069999999999999},"min":{"1min":0.65800000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":1.0169999999999999}]},{"osd":3,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92500000000000004,"15min":0.92100000000000004},"min":{"1min":0.621,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.012,"15min":1.018},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":0.94699999999999995}]},{"osd":4,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.48499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.2030000000000001,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.87,"15min":0.84599999999999997},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.365,"5min":1.5,"15min":1.5},"last":0.80000000000000004}]},{"osd":5,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.92400000000000004,"15min":0.90400000000000003},"min":{"1min":0.42599999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":0.88700000000000001},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.99099999999999999,"15min":0.98099999999999998},"min":{"1min":0.442,"5min":0.38,"15min":0.38},"max":{"1min":1.7470000000000001,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":0.91100000000000003}]},{"osd":7,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.141,"15min":1.149},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.6930000000000001,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.038},{"interface":"front","average":{"1min":1.05,"5min":1.129,"15min":1.1419999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.5860000000000001,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.2110000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574946,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:30:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64500000000000002,"15min":0.55700000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.253},"max":{"1min":1.1699999999999999,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":0.745},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.79800000000000004,"15min":0.64800000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29499999999999998},"max":{"1min":1.2030000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":1.0269999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:31:31 2024","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69699999999999995,"15min":0.64800000000000002},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.94199999999999995,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.99399999999999999},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.84699999999999998,"15min":0.73099999999999998},"min":{"1min":0.41799999999999998,"5min":0.41199999999999998,"15min":0.27900000000000003},"max":{"1min":1.226,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.95399999999999996}]},{"osd":3,"last update":"Sat Apr 13 07:31:21 2024","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.85999999999999999,"15min":0.77700000000000002},"min":{"1min":0.50900000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.1919999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.79500000000000004},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.88300000000000001},"min":{"1min":0.53700000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.2589999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":1.1679999999999999}]},{"osd":4,"last update":"Sat Apr 13 07:31:10 2024","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.194,"5min":1.349,"15min":1.349},"last":0.90800000000000003},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.96799999999999997,"15min":0.93500000000000005},"min":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.071}]},{"osd":5,"last update":"Sat Apr 13 07:31:04 2024","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.054,"15min":1.0409999999999999},"min":{"1min":0.73299999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.397,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":1.1100000000000001},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.98799999999999999,"15min":0.98699999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.228,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":1.0940000000000001}]},{"osd":6,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.01,"15min":1.018},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.3460000000000001,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":1.3540000000000001},{"interface":"front","average":{"1min":1.079,"5min":1.151,"15min":1.1559999999999999},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.409,"5min":1.498,"15min":1.498},"last":1.258}]},{"osd":7,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.1399999999999999,"15min":1.149},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.3009999999999999,"5min":1.482,"15min":1.482},"last":1.423},{"interface":"front","average":{"1min":1.097,"5min":1.173,"15min":1.1850000000000001},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.5649999999999999,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":1.3240000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738477,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32324,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712828,"statfs":{"total":95995035648,"available":95961935872,"internally_reserved":0,"allocated":1236992,"data_stored":857182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 13 07:30:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.69999999999999996,"15min":0.56200000000000006},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26500000000000001},"max":{"1min":1,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.72499999999999998},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.77100000000000002,"15min":0.61399999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28399999999999997},"max":{"1min":1.1930000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.95899999999999996}]},{"osd":2,"last update":"Sat Apr 13 07:31:30 2024","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.76900000000000002,"15min":0.69099999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.85399999999999998},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.79500000000000004,"15min":0.69599999999999995},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.93200000000000005,"5min":1.2,"15min":1.2},"last":0.91400000000000003}]},{"osd":3,"last update":"Sat Apr 13 07:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.78600000000000003,"15min":0.75800000000000001},"min":{"1min":0.41499999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.83899999999999997,"15min":0.81999999999999995},"min":{"1min":0.42899999999999999,"5min":0.378,"15min":0.378},"max":{"1min":1.155,"5min":1.2050000000000001,"15min":1.2629999999999999},"last":0.77100000000000002}]},{"osd":4,"last update":"Sat Apr 13 07:31:08 2024","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.97299999999999998,"15min":0.94499999999999995},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.1379999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":1.0049999999999999},{"interface":"front","average":{"1min":1.079,"5min":1.0429999999999999,"15min":1.006},"min":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"max":{"1min":1.4950000000000001,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.2789999999999999}]},{"osd":5,"last update":"Sat Apr 13 07:31:04 2024","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.0640000000000001,"15min":1.0209999999999999},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.748,"5min":1.748,"15min":1.748},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.024,"15min":0.996},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.619,"5min":1.619,"15min":1.619},"last":1.1830000000000001}]},{"osd":6,"last update":"Sat Apr 13 07:30:52 2024","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":1.163,"15min":1.175},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.5940000000000001,"5min":1.621,"15min":1.621},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.107,"15min":1.095},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.659,"5min":1.659,"15min":1.659},"last":1.2410000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:30:45 2024","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.212,"15min":1.228},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.542,"5min":1.647,"15min":1.647},"last":1.1140000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.2330000000000001,"15min":1.2410000000000001},"min":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.341}]}]},{"osd":2,"up_from":18,"seq":77309411416,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.72499999999999998,"15min":0.61599999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.78100000000000003,"15min":0.67600000000000005},"min":{"1min":0.38300000000000001,"5min":0.36299999999999999,"15min":0.35999999999999999},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.89600000000000002}]},{"osd":1,"last update":"Sat Apr 13 07:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.68600000000000005},"min":{"1min":0.439,"5min":0.41599999999999998,"15min":0.33200000000000002},"max":{"1min":1.1599999999999999,"5min":1.258,"15min":1.258},"last":0.94899999999999995},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.82999999999999996,"15min":0.71499999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.28899999999999998},"max":{"1min":1.2430000000000001,"5min":1.27,"15min":1.27},"last":0.73299999999999998}]},{"osd":3,"last update":"Sat Apr 13 07:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.872,"15min":0.79700000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2749999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.093},{"interface":"front","average":{"1min":0.872,"5min":0.89900000000000002,"15min":0.83599999999999997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.2350000000000001,"5min":1.389,"15min":1.389},"last":1.0529999999999999}]},{"osd":4,"last update":"Sat Apr 13 07:31:14 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.90600000000000003,"15min":0.91100000000000003},"min":{"1min":0.66300000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1719999999999999,"5min":1.341,"15min":1.341},"last":1.3169999999999999},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.048,"15min":1.03},"min":{"1min":0.83399999999999996,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3720000000000001,"5min":1.52,"15min":1.52},"last":1.155}]},{"osd":5,"last update":"Sat Apr 13 07:30:57 2024","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0389999999999999,"15min":1.054},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":1.383,"5min":1.53,"15min":1.53},"last":1.276},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.95399999999999996,"15min":0.96799999999999997},"min":{"1min":0.57499999999999996,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.202,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":1.234}]},{"osd":6,"last update":"Sat Apr 13 07:30:52 2024","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.1479999999999999,"15min":1.1779999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.431,"5min":1.788,"15min":1.788},"last":1.4399999999999999},{"interface":"front","average":{"1min":1.093,"5min":1.119,"15min":1.1319999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.411,"5min":1.5549999999999999,"15min":1.5549999999999999},"last":1.1679999999999999}]},{"osd":7,"last update":"Sat Apr 13 07:30:41 2024","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.1200000000000001,"15min":1.1200000000000001},"min":{"1min":0.56000000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.496,"5min":1.643,"15min":1.643},"last":1.3720000000000001},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1870000000000001,"15min":1.1910000000000001},"min":{"1min":0.75900000000000001,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":2.2130000000000001,"5min":2.2130000000000001,"15min":2.2130000000000001},"last":1.464}]}]},{"osd":3,"up_from":26,"seq":111669149774,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:31:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.67500000000000004,"15min":0.66300000000000003},"min":{"1min":0.27700000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.95899999999999996,"5min":1.196,"15min":1.196},"last":0.621},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.746,"15min":0.76200000000000001},"min":{"1min":0.315,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.0580000000000001,"5min":1.206,"15min":1.206},"last":1.056}]},{"osd":1,"last update":"Sat Apr 13 07:31:25 2024","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.71799999999999997,"15min":0.73499999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.0429999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.67000000000000004},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.79700000000000004,"15min":0.78500000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.198,"5min":1.2470000000000001,"15min":1.254},"last":0.90200000000000002}]},{"osd":2,"last update":"Sat Apr 13 07:31:25 2024","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.81699999999999995},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.83799999999999997,"15min":0.81399999999999995},"min":{"1min":0.36499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.294,"5min":1.294,"15min":1.3089999999999999},"last":0.97699999999999998}]},{"osd":4,"last update":"Sat Apr 13 07:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.91100000000000003,"15min":0.90800000000000003},"min":{"1min":0.52200000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.137,"5min":1.347,"15min":1.347},"last":1.1870000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.96899999999999997,"15min":0.94899999999999995},"min":{"1min":0.63900000000000001,"5min":0.495,"15min":0.495},"max":{"1min":1.228,"5min":1.389,"15min":1.389},"last":1.2929999999999999}]},{"osd":5,"last update":"Sat Apr 13 07:31:05 2024","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.02,"15min":1.026},"min":{"1min":0.67000000000000004,"5min":0.438,"15min":0.438},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.153},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.048,"15min":1.0529999999999999},"min":{"1min":0.70899999999999996,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.2050000000000001}]},{"osd":6,"last update":"Sat Apr 13 07:30:51 2024","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.056,"15min":1.0620000000000001},"min":{"1min":0.73199999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.3,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.3560000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0649999999999999,"15min":1.0569999999999999},"min":{"1min":0.71399999999999997,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.536,"5min":1.536,"15min":1.536},"last":1.3160000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:30:42 2024","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.0700000000000001,"15min":1.0629999999999999},"min":{"1min":0.73299999999999998,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.3620000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":1.401},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.121,"15min":1.1220000000000001},"min":{"1min":0.75,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":1.25}]}]},{"osd":4,"up_from":31,"seq":133143986244,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31744,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713408,"statfs":{"total":95995035648,"available":95962529792,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.84199999999999997,"15min":0.86399999999999999},"min":{"1min":0.60099999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.4570000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.92000000000000004,"15min":0.89400000000000002},"min":{"1min":0.63,"5min":0.432,"15min":0.432},"max":{"1min":1.355,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":1.01}]},{"osd":1,"last update":"Sat Apr 13 07:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.86899999999999999,"15min":0.88100000000000001},"min":{"1min":0.67900000000000005,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1699999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":1.071},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.90800000000000003,"15min":0.99399999999999999},"min":{"1min":0.58099999999999996,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.268,"5min":1.7190000000000001,"15min":1.7190000000000001},"last":1.0409999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:31:12 2024","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.89900000000000002,"15min":0.89500000000000002},"min":{"1min":0.78800000000000003,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.216,"5min":1.413,"15min":1.413},"last":1.125},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.93100000000000005,"15min":0.88700000000000001},"min":{"1min":0.59899999999999998,"5min":0.502,"15min":0.502},"max":{"1min":1.2989999999999999,"5min":1.48,"15min":1.48},"last":0.90600000000000003}]},{"osd":3,"last update":"Sat Apr 13 07:31:12 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.94999999999999996,"15min":0.92000000000000004},"min":{"1min":0.66600000000000004,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.4910000000000001,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":0.96199999999999997},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.94999999999999996,"15min":0.91200000000000003},"min":{"1min":0.63100000000000001,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":1.157}]},{"osd":5,"last update":"Sat Apr 13 07:31:00 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.84299999999999997,"15min":0.78300000000000003},"min":{"1min":0.504,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.81699999999999995,"15min":0.71499999999999997},"min":{"1min":0.53700000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.76,"5min":1.76,"15min":1.76},"last":0.98499999999999999}]},{"osd":6,"last update":"Sat Apr 13 07:30:57 2024","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":0.95399999999999996,"15min":0.93799999999999994},"min":{"1min":0.61699999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.675,"5min":1.675,"15min":1.675},"last":1.2110000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":0.996,"15min":0.98899999999999999},"min":{"1min":0.53600000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.734,"5min":1.734,"15min":1.734},"last":0.83499999999999996}]},{"osd":7,"last update":"Sat Apr 13 07:30:46 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.98799999999999999,"15min":0.97599999999999998},"min":{"1min":0.44500000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.8380000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":0.871},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.95099999999999996,"15min":0.94599999999999995},"min":{"1min":0.42699999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.5740000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.92300000000000004}]}]},{"osd":5,"up_from":39,"seq":167503724602,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.88400000000000001,"15min":0.88800000000000001},"min":{"1min":0.61699999999999999,"5min":0.435,"15min":0.435},"max":{"1min":1.1679999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":0.84399999999999997},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.89500000000000002,"15min":0.879},"min":{"1min":0.629,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.22,"5min":1.266,"15min":1.266},"last":1.161}]},{"osd":1,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.92700000000000005,"15min":0.90200000000000002},"min":{"1min":0.622,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":1.032},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.90400000000000003,"15min":0.91500000000000004},"min":{"1min":0.67200000000000004,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.1180000000000001,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":1.2669999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.95899999999999996,"15min":0.90900000000000003},"min":{"1min":0.69799999999999995,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.91000000000000003},{"interface":"front","average":{"1min":1.022,"5min":0.95799999999999996,"15min":0.94499999999999995},"min":{"1min":0.70399999999999996,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.95499999999999996}]},{"osd":3,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.94299999999999995,"15min":0.93300000000000005},"min":{"1min":0.65500000000000003,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.202,"5min":1.319,"15min":1.319},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.1279999999999999,"5min":0.99399999999999999,"15min":0.97399999999999998},"min":{"1min":0.77100000000000002,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":1.0880000000000001}]},{"osd":4,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.80000000000000004,"15min":0.76900000000000002},"min":{"1min":0.47399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.056},{"interface":"front","average":{"1min":1.115,"5min":0.88700000000000001,"15min":0.81499999999999995},"min":{"1min":0.45300000000000001,"5min":0.318,"15min":0.318},"max":{"1min":1.7350000000000001,"5min":1.7350000000000001,"15min":1.7350000000000001},"last":1.2350000000000001}]},{"osd":6,"last update":"Sat Apr 13 07:30:50 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":0.88300000000000001,"15min":0.81100000000000005},"min":{"1min":0.54600000000000004,"5min":0.37,"15min":0.37},"max":{"1min":1.446,"5min":1.446,"15min":1.446},"last":0.995},{"interface":"front","average":{"1min":1.198,"5min":0.89400000000000002,"15min":0.80400000000000005},"min":{"1min":0.499,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.7909999999999999,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.1930000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:30:44 2024","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":0.94799999999999995,"15min":0.92600000000000005},"min":{"1min":0.56499999999999995,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":1.3380000000000001},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.089,"15min":1.0780000000000001},"min":{"1min":0.59499999999999997,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.4670000000000001,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":1.3129999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-13T07:31:37.968 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-13T07:31:38.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:37 smithi203 bash[23480]: cluster 2024-04-13T07:31:36.881074+0000 mgr.y (mgr.14373) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:38.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:37 smithi203 bash[23480]: audit 2024-04-13T07:31:37.431832+0000 mgr.y (mgr.14373) 94 : audit 0 from='client.24449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:31:38.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:37 smithi089 bash[21353]: cluster 2024-04-13T07:31:36.881074+0000 mgr.y (mgr.14373) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:38.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:37 smithi089 bash[21353]: audit 2024-04-13T07:31:37.431832+0000 mgr.y (mgr.14373) 94 : audit 0 from='client.24449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:31:38.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:37 smithi089 bash[31522]: cluster 2024-04-13T07:31:36.881074+0000 mgr.y (mgr.14373) 93 : cluster 0 pgmap v66: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:38.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:37 smithi089 bash[31522]: audit 2024-04-13T07:31:37.431832+0000 mgr.y (mgr.14373) 94 : audit 0 from='client.24449 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:31:39.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:38 smithi203 bash[23480]: audit 2024-04-13T07:31:38.930758+0000 mon.a (mon.0) 783 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:39.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:38 smithi089 bash[21353]: audit 2024-04-13T07:31:38.930758+0000 mon.a (mon.0) 783 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:39.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:38 smithi089 bash[31522]: audit 2024-04-13T07:31:38.930758+0000 mon.a (mon.0) 783 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:40.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:39 smithi203 bash[23480]: cluster 2024-04-13T07:31:38.881615+0000 mgr.y (mgr.14373) 95 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:40.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:39 smithi089 bash[31522]: cluster 2024-04-13T07:31:38.881615+0000 mgr.y (mgr.14373) 95 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:40.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:39 smithi089 bash[21353]: cluster 2024-04-13T07:31:38.881615+0000 mgr.y (mgr.14373) 95 : cluster 0 pgmap v67: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:42.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:41 smithi203 bash[23480]: cluster 2024-04-13T07:31:40.882224+0000 mgr.y (mgr.14373) 96 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:42.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:41 smithi089 bash[31522]: cluster 2024-04-13T07:31:40.882224+0000 mgr.y (mgr.14373) 96 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:42.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:41 smithi089 bash[21353]: cluster 2024-04-13T07:31:40.882224+0000 mgr.y (mgr.14373) 96 : cluster 0 pgmap v68: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:42.694 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:44.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:43 smithi089 bash[21353]: cluster 2024-04-13T07:31:42.882790+0000 mgr.y (mgr.14373) 97 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:44.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:43 smithi089 bash[31522]: cluster 2024-04-13T07:31:42.882790+0000 mgr.y (mgr.14373) 97 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:44.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:43 smithi203 bash[23480]: cluster 2024-04-13T07:31:42.882790+0000 mgr.y (mgr.14373) 97 : cluster 0 pgmap v69: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:45.238 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:31:45.241 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-04-13T07:31:45.798 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":70,"stamp":"2024-04-13T07:31:44.883015+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":235200,"kb_used_data":6752,"kb_used_omap":12,"kb_used_meta":228339,"kb_avail":749726016,"statfs":{"total":767960285184,"available":767719440384,"internally_reserved":0,"allocated":6914048,"data_stored":3904681,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":233819705},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003454"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.882284+0000","last_change":"2024-04-13T07:30:21.470196+0000","last_active":"2024-04-13T07:30:24.882284+0000","last_peered":"2024-04-13T07:30:24.882284+0000","last_clean":"2024-04-13T07:30:24.882284+0000","last_became_active":"2024-04-13T07:30:21.469340+0000","last_became_peered":"2024-04-13T07:30:21.469340+0000","last_unstale":"2024-04-13T07:30:24.882284+0000","last_undegraded":"2024-04-13T07:30:24.882284+0000","last_fullsized":"2024-04-13T07:30:24.882284+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:32:48.529436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.876804+0000","last_change":"2024-04-13T07:30:22.257370+0000","last_active":"2024-04-13T07:30:24.876804+0000","last_peered":"2024-04-13T07:30:24.876804+0000","last_clean":"2024-04-13T07:30:24.876804+0000","last_became_active":"2024-04-13T07:30:22.256904+0000","last_became_peered":"2024-04-13T07:30:22.256904+0000","last_unstale":"2024-04-13T07:30:24.876804+0000","last_undegraded":"2024-04-13T07:30:24.876804+0000","last_fullsized":"2024-04-13T07:30:24.876804+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:09:42.362257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.883082+0000","last_change":"2024-04-13T07:30:22.258054+0000","last_active":"2024-04-13T07:30:24.883082+0000","last_peered":"2024-04-13T07:30:24.883082+0000","last_clean":"2024-04-13T07:30:24.883082+0000","last_became_active":"2024-04-13T07:30:22.256961+0000","last_became_peered":"2024-04-13T07:30:22.256961+0000","last_unstale":"2024-04-13T07:30:24.883082+0000","last_undegraded":"2024-04-13T07:30:24.883082+0000","last_fullsized":"2024-04-13T07:30:24.883082+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:13:36.989961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.882984+0000","last_change":"2024-04-13T07:30:21.472711+0000","last_active":"2024-04-13T07:30:24.882984+0000","last_peered":"2024-04-13T07:30:24.882984+0000","last_clean":"2024-04-13T07:30:24.882984+0000","last_became_active":"2024-04-13T07:30:21.471979+0000","last_became_peered":"2024-04-13T07:30:21.471979+0000","last_unstale":"2024-04-13T07:30:24.882984+0000","last_undegraded":"2024-04-13T07:30:24.882984+0000","last_fullsized":"2024-04-13T07:30:24.882984+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:27:07.929194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":260,"reported_epoch":64,"state":"active+clean","last_fresh":"2024-04-13T07:30:29.895446+0000","last_change":"2024-04-13T07:28:43.834029+0000","last_active":"2024-04-13T07:30:29.895446+0000","last_peered":"2024-04-13T07:30:29.895446+0000","last_clean":"2024-04-13T07:30:29.895446+0000","last_became_active":"2024-04-13T07:28:43.833192+0000","last_became_peered":"2024-04-13T07:28:43.833192+0000","last_unstale":"2024-04-13T07:30:29.895446+0000","last_undegraded":"2024-04-13T07:30:29.895446+0000","last_fullsized":"2024-04-13T07:30:29.895446+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_clean_scrub_stamp":"2024-04-13T07:24:24.657602+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:32:17.256871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.881728+0000","last_change":"2024-04-13T07:30:21.467433+0000","last_active":"2024-04-13T07:30:24.881728+0000","last_peered":"2024-04-13T07:30:24.881728+0000","last_clean":"2024-04-13T07:30:24.881728+0000","last_became_active":"2024-04-13T07:30:21.466602+0000","last_became_peered":"2024-04-13T07:30:21.466602+0000","last_unstale":"2024-04-13T07:30:24.881728+0000","last_undegraded":"2024-04-13T07:30:24.881728+0000","last_fullsized":"2024-04-13T07:30:24.881728+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:29:15.004301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.880732+0000","last_change":"2024-04-13T07:30:21.471551+0000","last_active":"2024-04-13T07:30:24.880732+0000","last_peered":"2024-04-13T07:30:24.880732+0000","last_clean":"2024-04-13T07:30:24.880732+0000","last_became_active":"2024-04-13T07:30:21.470542+0000","last_became_peered":"2024-04-13T07:30:21.470542+0000","last_unstale":"2024-04-13T07:30:24.880732+0000","last_undegraded":"2024-04-13T07:30:24.880732+0000","last_fullsized":"2024-04-13T07:30:24.880732+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:54:43.387491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.881691+0000","last_change":"2024-04-13T07:30:21.469386+0000","last_active":"2024-04-13T07:30:24.881691+0000","last_peered":"2024-04-13T07:30:24.881691+0000","last_clean":"2024-04-13T07:30:24.881691+0000","last_became_active":"2024-04-13T07:30:21.468314+0000","last_became_peered":"2024-04-13T07:30:21.468314+0000","last_unstale":"2024-04-13T07:30:24.881691+0000","last_undegraded":"2024-04-13T07:30:24.881691+0000","last_fullsized":"2024-04-13T07:30:24.881691+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:51:04.281850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":63,"state":"active+clean","last_fresh":"2024-04-13T07:30:24.876767+0000","last_change":"2024-04-13T07:30:21.471173+0000","last_active":"2024-04-13T07:30:24.876767+0000","last_peered":"2024-04-13T07:30:24.876767+0000","last_clean":"2024-04-13T07:30:24.876767+0000","last_became_active":"2024-04-13T07:30:21.470193+0000","last_became_peered":"2024-04-13T07:30:21.470193+0000","last_unstale":"2024-04-13T07:30:24.876767+0000","last_undegraded":"2024-04-13T07:30:24.876767+0000","last_fullsized":"2024-04-13T07:30:24.876767+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:13:17.367790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201318,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32324,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712828,"statfs":{"total":95995035648,"available":95961935872,"internally_reserved":0,"allocated":1236992,"data_stored":857182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.86399999999999999,"15min":0.871},"min":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"max":{"1min":1.2250000000000001,"5min":1.3,"15min":1.3},"last":1.1579999999999999},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.92300000000000004,"15min":0.93300000000000005},"min":{"1min":0.59899999999999998,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.2030000000000001,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.98499999999999999}]},{"osd":1,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":0.88600000000000001,"15min":0.88},"min":{"1min":0.59199999999999997,"5min":0.56200000000000006,"15min":0.56200000000000006},"max":{"1min":1.302,"5min":1.302,"15min":1.302},"last":0.93500000000000005},{"interface":"front","average":{"1min":0.95699999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"min":{"1min":0.58099999999999996,"5min":0.53400000000000003,"15min":0.53400000000000003},"max":{"1min":1.282,"5min":1.3819999999999999,"15min":1.3819999999999999},"last":1.3180000000000001}]},{"osd":2,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.92000000000000004,"15min":0.92100000000000004},"min":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"max":{"1min":1.272,"5min":1.272,"15min":1.272},"last":1.03},{"interface":"front","average":{"1min":0.96999999999999997,"5min":1.01,"15min":1.016},"min":{"1min":0.66500000000000004,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.2529999999999999,"5min":1.367,"15min":1.367},"last":1.054}]},{"osd":3,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.94199999999999995,"15min":0.94199999999999995},"min":{"1min":0.60499999999999998,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.401,"5min":1.401,"15min":1.401},"last":0.85299999999999998},{"interface":"front","average":{"1min":1.016,"5min":1.036,"15min":1.04},"min":{"1min":0.61299999999999999,"5min":0.501,"15min":0.501},"max":{"1min":1.355,"5min":1.355,"15min":1.355},"last":1.304}]},{"osd":4,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.86499999999999999,"5min":0.87,"15min":0.871},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3240000000000001,"5min":1.3240000000000001,"15min":1.3240000000000001},"last":1.0940000000000001},{"interface":"front","average":{"1min":0.89600000000000002,"5min":0.92000000000000004,"15min":0.92400000000000004},"min":{"1min":0.47399999999999998,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.2989999999999999,"5min":1.3320000000000001,"15min":1.3320000000000001},"last":1.1259999999999999}]},{"osd":5,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":0.95099999999999996,"5min":0.98599999999999999,"15min":0.99199999999999999},"min":{"1min":0.49099999999999999,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":1.3420000000000001,"5min":1.4019999999999999,"15min":1.4019999999999999},"last":1.2569999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.03,"15min":1.04},"min":{"1min":0.498,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.262,"5min":1.4810000000000001,"15min":1.4810000000000001},"last":1.3899999999999999}]},{"osd":6,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.0900000000000001,"15min":1.0960000000000001},"min":{"1min":0.52800000000000002,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.3340000000000001,"5min":1.536,"15min":1.536},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.008,"5min":1.0840000000000001,"15min":1.0960000000000001},"min":{"1min":0.56799999999999995,"5min":0.434,"15min":0.434},"max":{"1min":1.3640000000000001,"5min":1.633,"15min":1.633},"last":1.1950000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462960,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28220,"kb_used_data":1200,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716932,"statfs":{"total":95995035648,"available":95966138368,"internally_reserved":0,"allocated":1228800,"data_stored":856578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":0.86799999999999999,"15min":0.84799999999999998},"min":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.5109999999999999,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":1.0600000000000001},{"interface":"front","average":{"1min":0.875,"5min":0.88400000000000001,"15min":0.88},"min":{"1min":0.55300000000000005,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.234,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.0209999999999999}]},{"osd":1,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.90100000000000002,"5min":0.92100000000000004,"15min":0.92200000000000004},"min":{"1min":0.56899999999999995,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.327,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.97599999999999998},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.93100000000000005,"15min":0.93300000000000005},"min":{"1min":0.60599999999999998,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.4470000000000001,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.75800000000000001}]},{"osd":2,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.91600000000000004,"5min":0.91700000000000004,"15min":0.90000000000000002},"min":{"1min":0.57299999999999995,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.2849999999999999,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.89900000000000002,"5min":0.98899999999999999,"15min":1.0069999999999999},"min":{"1min":0.65800000000000003,"5min":0.60699999999999998,"15min":0.60699999999999998},"max":{"1min":1.268,"5min":1.268,"15min":1.268},"last":0.91900000000000004}]},{"osd":3,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.92700000000000005,"5min":0.92500000000000004,"15min":0.92100000000000004},"min":{"1min":0.621,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":1.121},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.012,"15min":1.018},"min":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.5660000000000001,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.161}]},{"osd":4,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":0.89200000000000002,"15min":0.89200000000000002},"min":{"1min":0.48499999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.2030000000000001,"5min":1.3129999999999999,"15min":1.3129999999999999},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.88800000000000001,"5min":0.87,"15min":0.84599999999999997},"min":{"1min":0.35399999999999998,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.365,"5min":1.5,"15min":1.5},"last":1.181}]},{"osd":5,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.92400000000000004,"15min":0.90400000000000003},"min":{"1min":0.42599999999999999,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.423,"5min":1.423,"15min":1.423},"last":1.1459999999999999},{"interface":"front","average":{"1min":0.95999999999999996,"5min":0.99099999999999999,"15min":0.98099999999999998},"min":{"1min":0.442,"5min":0.38,"15min":0.38},"max":{"1min":1.7470000000000001,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":1.274}]},{"osd":7,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.141,"15min":1.149},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.6930000000000001,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.2310000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.129,"15min":1.1419999999999999},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":1.5860000000000001,"5min":1.5860000000000001,"15min":1.5860000000000001},"last":1.1000000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574947,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:30:49 2024","interfaces":[{"interface":"back","average":{"1min":0.59999999999999998,"5min":0.64500000000000002,"15min":0.55700000000000005},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.253},"max":{"1min":1.1699999999999999,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.79800000000000004,"15min":0.64800000000000002},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.29499999999999998},"max":{"1min":1.2030000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"last":0.86799999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:31:31 2024","interfaces":[{"interface":"back","average":{"1min":0.69799999999999995,"5min":0.69699999999999995,"15min":0.64800000000000002},"min":{"1min":0.318,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.94199999999999995,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.89100000000000001,"5min":0.84699999999999998,"15min":0.73099999999999998},"min":{"1min":0.41799999999999998,"5min":0.41199999999999998,"15min":0.27900000000000003},"max":{"1min":1.226,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.79700000000000004}]},{"osd":3,"last update":"Sat Apr 13 07:31:21 2024","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.85999999999999999,"15min":0.77700000000000002},"min":{"1min":0.50900000000000001,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.1919999999999999,"5min":1.3939999999999999,"15min":1.3939999999999999},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.98499999999999999,"5min":0.98499999999999999,"15min":0.88300000000000001},"min":{"1min":0.53700000000000003,"5min":0.46800000000000003,"15min":0.46800000000000003},"max":{"1min":1.2589999999999999,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.997}]},{"osd":4,"last update":"Sat Apr 13 07:31:10 2024","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":0.91300000000000003,"15min":0.91300000000000003},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":1.194,"5min":1.349,"15min":1.349},"last":0.95999999999999996},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.96799999999999997,"15min":0.93500000000000005},"min":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"max":{"1min":1.4350000000000001,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":1.2010000000000001}]},{"osd":5,"last update":"Sat Apr 13 07:31:04 2024","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.054,"15min":1.0409999999999999},"min":{"1min":0.73299999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":1.397,"5min":1.5469999999999999,"15min":1.5469999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.97599999999999998,"5min":0.98799999999999999,"15min":0.98699999999999999},"min":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.228,"5min":1.3180000000000001,"15min":1.3180000000000001},"last":1.0469999999999999}]},{"osd":6,"last update":"Sat Apr 13 07:30:55 2024","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.01,"15min":1.018},"min":{"1min":0.65400000000000003,"5min":0.65400000000000003,"15min":0.65400000000000003},"max":{"1min":1.3460000000000001,"5min":1.4059999999999999,"15min":1.4059999999999999},"last":1.1160000000000001},{"interface":"front","average":{"1min":1.079,"5min":1.151,"15min":1.1559999999999999},"min":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.409,"5min":1.498,"15min":1.498},"last":1.153}]},{"osd":7,"last update":"Sat Apr 13 07:30:43 2024","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":1.1399999999999999,"15min":1.149},"min":{"1min":0.68500000000000005,"5min":0.68500000000000005,"15min":0.68500000000000005},"max":{"1min":1.3009999999999999,"5min":1.482,"15min":1.482},"last":1.228},{"interface":"front","average":{"1min":1.097,"5min":1.173,"15min":1.1850000000000001},"min":{"1min":0.624,"5min":0.624,"15min":0.624},"max":{"1min":1.5649999999999999,"5min":1.6499999999999999,"15min":1.6499999999999999},"last":1.0900000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738478,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32324,"kb_used_data":1208,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712828,"statfs":{"total":95995035648,"available":95961935872,"internally_reserved":0,"allocated":1236992,"data_stored":857182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 13 07:30:48 2024","interfaces":[{"interface":"back","average":{"1min":0.63900000000000001,"5min":0.69999999999999996,"15min":0.56200000000000006},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.26500000000000001},"max":{"1min":1,"5min":1.1910000000000001,"15min":1.1910000000000001},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.77100000000000002,"15min":0.61399999999999999},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.28399999999999997},"max":{"1min":1.1930000000000001,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":1.004}]},{"osd":2,"last update":"Sat Apr 13 07:31:30 2024","interfaces":[{"interface":"back","average":{"1min":0.70099999999999996,"5min":0.76900000000000002,"15min":0.69099999999999995},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":1.1359999999999999,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.70399999999999996,"5min":0.79500000000000004,"15min":0.69599999999999995},"min":{"1min":0.32600000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.93200000000000005,"5min":1.2,"15min":1.2},"last":0.83399999999999996}]},{"osd":3,"last update":"Sat Apr 13 07:31:24 2024","interfaces":[{"interface":"back","average":{"1min":0.76600000000000001,"5min":0.78600000000000003,"15min":0.75800000000000001},"min":{"1min":0.41499999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.3979999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.96399999999999997},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.83899999999999997,"15min":0.81999999999999995},"min":{"1min":0.42899999999999999,"5min":0.378,"15min":0.378},"max":{"1min":1.155,"5min":1.2050000000000001,"15min":1.2629999999999999},"last":1.0489999999999999}]},{"osd":4,"last update":"Sat Apr 13 07:31:08 2024","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.97299999999999998,"15min":0.94499999999999995},"min":{"1min":0.57599999999999996,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.1379999999999999,"5min":1.3029999999999999,"15min":1.3029999999999999},"last":1.1140000000000001},{"interface":"front","average":{"1min":1.079,"5min":1.0429999999999999,"15min":1.006},"min":{"1min":0.70799999999999996,"5min":0.70799999999999996,"15min":0.70799999999999996},"max":{"1min":1.4950000000000001,"5min":1.5249999999999999,"15min":1.5249999999999999},"last":1.0820000000000001}]},{"osd":5,"last update":"Sat Apr 13 07:31:04 2024","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.0640000000000001,"15min":1.0209999999999999},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":1.748,"5min":1.748,"15min":1.748},"last":1.4259999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.024,"15min":0.996},"min":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.619,"5min":1.619,"15min":1.619},"last":1.4610000000000001}]},{"osd":6,"last update":"Sat Apr 13 07:30:52 2024","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":1.163,"15min":1.175},"min":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.5940000000000001,"5min":1.621,"15min":1.621},"last":1.3300000000000001},{"interface":"front","average":{"1min":1.1539999999999999,"5min":1.107,"15min":1.095},"min":{"1min":0.71099999999999997,"5min":0.71099999999999997,"15min":0.71099999999999997},"max":{"1min":1.659,"5min":1.659,"15min":1.659},"last":1.2909999999999999}]},{"osd":7,"last update":"Sat Apr 13 07:30:45 2024","interfaces":[{"interface":"back","average":{"1min":1.113,"5min":1.212,"15min":1.228},"min":{"1min":0.55300000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.542,"5min":1.647,"15min":1.647},"last":1.3700000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.2330000000000001,"15min":1.2410000000000001},"min":{"1min":0.68999999999999995,"5min":0.68999999999999995,"15min":0.68999999999999995},"max":{"1min":1.6970000000000001,"5min":1.6970000000000001,"15min":1.6970000000000001},"last":1.1990000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411417,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.69699999999999995,"5min":0.72499999999999998,"15min":0.61599999999999999},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":1.2090000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.80100000000000005,"5min":0.78100000000000003,"15min":0.67600000000000005},"min":{"1min":0.38300000000000001,"5min":0.36299999999999999,"15min":0.35999999999999999},"max":{"1min":1.296,"5min":1.296,"15min":1.296},"last":0.91400000000000003}]},{"osd":1,"last update":"Sat Apr 13 07:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.78400000000000003,"15min":0.68600000000000005},"min":{"1min":0.439,"5min":0.41599999999999998,"15min":0.33200000000000002},"max":{"1min":1.1599999999999999,"5min":1.258,"15min":1.258},"last":1.0620000000000001},{"interface":"front","average":{"1min":0.80300000000000005,"5min":0.82999999999999996,"15min":0.71499999999999997},"min":{"1min":0.309,"5min":0.309,"15min":0.28899999999999998},"max":{"1min":1.2430000000000001,"5min":1.27,"15min":1.27},"last":1.1439999999999999}]},{"osd":3,"last update":"Sat Apr 13 07:31:28 2024","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.872,"15min":0.79700000000000004},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.2749999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.872,"5min":0.89900000000000002,"15min":0.83599999999999997},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.2350000000000001,"5min":1.389,"15min":1.389},"last":1.4830000000000001}]},{"osd":4,"last update":"Sat Apr 13 07:31:14 2024","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.90600000000000003,"15min":0.91100000000000003},"min":{"1min":0.66300000000000003,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1719999999999999,"5min":1.341,"15min":1.341},"last":1.1040000000000001},{"interface":"front","average":{"1min":1.0860000000000001,"5min":1.048,"15min":1.03},"min":{"1min":0.83399999999999996,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.3720000000000001,"5min":1.52,"15min":1.52},"last":1.2350000000000001}]},{"osd":5,"last update":"Sat Apr 13 07:30:57 2024","interfaces":[{"interface":"back","average":{"1min":0.98899999999999999,"5min":1.0389999999999999,"15min":1.054},"min":{"1min":0.498,"5min":0.498,"15min":0.498},"max":{"1min":1.383,"5min":1.53,"15min":1.53},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.91400000000000003,"5min":0.95399999999999996,"15min":0.96799999999999997},"min":{"1min":0.57499999999999996,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.202,"5min":1.4850000000000001,"15min":1.4850000000000001},"last":0.98299999999999998}]},{"osd":6,"last update":"Sat Apr 13 07:30:52 2024","interfaces":[{"interface":"back","average":{"1min":1.093,"5min":1.1479999999999999,"15min":1.1779999999999999},"min":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.431,"5min":1.788,"15min":1.788},"last":1.302},{"interface":"front","average":{"1min":1.093,"5min":1.119,"15min":1.1319999999999999},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.411,"5min":1.5549999999999999,"15min":1.5549999999999999},"last":1.407}]},{"osd":7,"last update":"Sat Apr 13 07:30:41 2024","interfaces":[{"interface":"back","average":{"1min":1.1200000000000001,"5min":1.1200000000000001,"15min":1.1200000000000001},"min":{"1min":0.56000000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.496,"5min":1.643,"15min":1.643},"last":1.3520000000000001},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1870000000000001,"15min":1.1910000000000001},"min":{"1min":0.75900000000000001,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":2.2130000000000001,"5min":2.2130000000000001,"15min":2.2130000000000001},"last":1.5600000000000001}]}]},{"osd":3,"up_from":26,"seq":111669149775,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27648,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717504,"statfs":{"total":95995035648,"available":95966724096,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:31:25 2024","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.67500000000000004,"15min":0.66300000000000003},"min":{"1min":0.27700000000000002,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.95899999999999996,"5min":1.196,"15min":1.196},"last":0.624},{"interface":"front","average":{"1min":0.67700000000000005,"5min":0.746,"15min":0.76200000000000001},"min":{"1min":0.315,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.0580000000000001,"5min":1.206,"15min":1.206},"last":0.96599999999999997}]},{"osd":1,"last update":"Sat Apr 13 07:31:25 2024","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.71799999999999997,"15min":0.73499999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":1.0429999999999999,"5min":1.0509999999999999,"15min":1.0509999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.81200000000000006,"5min":0.79700000000000004,"15min":0.78500000000000003},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":1.198,"5min":1.2470000000000001,"15min":1.254},"last":0.74199999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:31:25 2024","interfaces":[{"interface":"back","average":{"1min":0.82599999999999996,"5min":0.83599999999999997,"15min":0.83599999999999997},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.3620000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.85899999999999999},{"interface":"front","average":{"1min":0.82999999999999996,"5min":0.83799999999999997,"15min":0.81399999999999995},"min":{"1min":0.36499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.294,"5min":1.294,"15min":1.3089999999999999},"last":0.77100000000000002}]},{"osd":4,"last update":"Sat Apr 13 07:31:11 2024","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":0.91100000000000003,"15min":0.90800000000000003},"min":{"1min":0.52200000000000002,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.137,"5min":1.347,"15min":1.347},"last":0.89700000000000002},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.96899999999999997,"15min":0.94899999999999995},"min":{"1min":0.63900000000000001,"5min":0.495,"15min":0.495},"max":{"1min":1.228,"5min":1.389,"15min":1.389},"last":1.115}]},{"osd":5,"last update":"Sat Apr 13 07:31:05 2024","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.02,"15min":1.026},"min":{"1min":0.67000000000000004,"5min":0.438,"15min":0.438},"max":{"1min":1.46,"5min":1.46,"15min":1.46},"last":1.099},{"interface":"front","average":{"1min":1.0289999999999999,"5min":1.048,"15min":1.0529999999999999},"min":{"1min":0.70899999999999996,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.4419999999999999,"5min":1.4419999999999999,"15min":1.4419999999999999},"last":1.0389999999999999}]},{"osd":6,"last update":"Sat Apr 13 07:30:51 2024","interfaces":[{"interface":"back","average":{"1min":1.044,"5min":1.056,"15min":1.0620000000000001},"min":{"1min":0.73199999999999998,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.3,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.0640000000000001,"5min":1.0649999999999999,"15min":1.0569999999999999},"min":{"1min":0.71399999999999997,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.536,"5min":1.536,"15min":1.536},"last":0.92700000000000005}]},{"osd":7,"last update":"Sat Apr 13 07:30:42 2024","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.0700000000000001,"15min":1.0629999999999999},"min":{"1min":0.73299999999999998,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.3620000000000001,"5min":1.4690000000000001,"15min":1.4690000000000001},"last":1.157},{"interface":"front","average":{"1min":1.1140000000000001,"5min":1.121,"15min":1.1220000000000001},"min":{"1min":0.75,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":1.1870000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986245,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31744,"kb_used_data":628,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713408,"statfs":{"total":95995035648,"available":95962529792,"internally_reserved":0,"allocated":643072,"data_stored":266814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.91500000000000004,"5min":0.84199999999999997,"15min":0.86399999999999999},"min":{"1min":0.60099999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.4570000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.92000000000000004,"15min":0.89400000000000002},"min":{"1min":0.63,"5min":0.432,"15min":0.432},"max":{"1min":1.355,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.94799999999999995}]},{"osd":1,"last update":"Sat Apr 13 07:31:12 2024","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.86899999999999999,"15min":0.88100000000000001},"min":{"1min":0.67900000000000005,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.1699999999999999,"5min":1.6819999999999999,"15min":1.6819999999999999},"last":0.80900000000000005},{"interface":"front","average":{"1min":0.93400000000000005,"5min":0.90800000000000003,"15min":0.99399999999999999},"min":{"1min":0.58099999999999996,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.268,"5min":1.7190000000000001,"15min":1.7190000000000001},"last":0.99199999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:31:12 2024","interfaces":[{"interface":"back","average":{"1min":1.0029999999999999,"5min":0.89900000000000002,"15min":0.89500000000000002},"min":{"1min":0.78800000000000003,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.216,"5min":1.413,"15min":1.413},"last":1.0449999999999999},{"interface":"front","average":{"1min":0.94099999999999995,"5min":0.93100000000000005,"15min":0.88700000000000001},"min":{"1min":0.59899999999999998,"5min":0.502,"15min":0.502},"max":{"1min":1.2989999999999999,"5min":1.48,"15min":1.48},"last":0.83299999999999996}]},{"osd":3,"last update":"Sat Apr 13 07:31:12 2024","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.94999999999999996,"15min":0.92000000000000004},"min":{"1min":0.66600000000000004,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.4910000000000001,"5min":1.7849999999999999,"15min":1.7849999999999999},"last":1.1679999999999999},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.94999999999999996,"15min":0.91200000000000003},"min":{"1min":0.63100000000000001,"5min":0.57399999999999995,"15min":0.57399999999999995},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":1.02}]},{"osd":5,"last update":"Sat Apr 13 07:31:00 2024","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.84299999999999997,"15min":0.78300000000000003},"min":{"1min":0.504,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.5600000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":1.1220000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":0.81699999999999995,"15min":0.71499999999999997},"min":{"1min":0.53700000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":1.76,"5min":1.76,"15min":1.76},"last":0.71099999999999997}]},{"osd":6,"last update":"Sat Apr 13 07:30:57 2024","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":0.95399999999999996,"15min":0.93799999999999994},"min":{"1min":0.61699999999999999,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":1.675,"5min":1.675,"15min":1.675},"last":0.89100000000000001},{"interface":"front","average":{"1min":1.0920000000000001,"5min":0.996,"15min":0.98899999999999999},"min":{"1min":0.53600000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.734,"5min":1.734,"15min":1.734},"last":1.0660000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:30:46 2024","interfaces":[{"interface":"back","average":{"1min":1.0569999999999999,"5min":0.98799999999999999,"15min":0.97599999999999998},"min":{"1min":0.44500000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.8380000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":1.0880000000000001},{"interface":"front","average":{"1min":0.98099999999999998,"5min":0.95099999999999996,"15min":0.94599999999999995},"min":{"1min":0.42699999999999999,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":1.5740000000000001,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.91300000000000003}]}]},{"osd":5,"up_from":39,"seq":167503724603,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27644,"kb_used_data":624,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717508,"statfs":{"total":95995035648,"available":95966728192,"internally_reserved":0,"allocated":638976,"data_stored":266483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.88400000000000001,"15min":0.88800000000000001},"min":{"1min":0.61699999999999999,"5min":0.435,"15min":0.435},"max":{"1min":1.1679999999999999,"5min":1.2529999999999999,"15min":1.2529999999999999},"last":1.1739999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.89500000000000002,"15min":0.879},"min":{"1min":0.629,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.22,"5min":1.266,"15min":1.266},"last":1.0049999999999999}]},{"osd":1,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0069999999999999,"5min":0.92700000000000005,"15min":0.90200000000000002},"min":{"1min":0.622,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.65700000000000003},{"interface":"front","average":{"1min":0.89700000000000002,"5min":0.90400000000000003,"15min":0.91500000000000004},"min":{"1min":0.67200000000000004,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.1180000000000001,"5min":1.2390000000000001,"15min":1.2390000000000001},"last":0.754}]},{"osd":2,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":0.95899999999999996,"15min":0.90900000000000003},"min":{"1min":0.69799999999999995,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.343,"5min":1.343,"15min":1.343},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.022,"5min":0.95799999999999996,"15min":0.94499999999999995},"min":{"1min":0.70399999999999996,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.83099999999999996}]},{"osd":3,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":0.94299999999999995,"15min":0.93300000000000005},"min":{"1min":0.65500000000000003,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.202,"5min":1.319,"15min":1.319},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.1279999999999999,"5min":0.99399999999999999,"15min":0.97399999999999998},"min":{"1min":0.77100000000000002,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.6890000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":1.048}]},{"osd":4,"last update":"Sat Apr 13 07:31:02 2024","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":0.80000000000000004,"15min":0.76900000000000002},"min":{"1min":0.47399999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.3460000000000001,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":1.141},{"interface":"front","average":{"1min":1.115,"5min":0.88700000000000001,"15min":0.81499999999999995},"min":{"1min":0.45300000000000001,"5min":0.318,"15min":0.318},"max":{"1min":1.7350000000000001,"5min":1.7350000000000001,"15min":1.7350000000000001},"last":1.214}]},{"osd":6,"last update":"Sat Apr 13 07:30:50 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":0.88300000000000001,"15min":0.81100000000000005},"min":{"1min":0.54600000000000004,"5min":0.37,"15min":0.37},"max":{"1min":1.446,"5min":1.446,"15min":1.446},"last":1.107},{"interface":"front","average":{"1min":1.198,"5min":0.89400000000000002,"15min":0.80400000000000005},"min":{"1min":0.499,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.7909999999999999,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.2529999999999999}]},{"osd":7,"last update":"Sat Apr 13 07:30:44 2024","interfaces":[{"interface":"back","average":{"1min":1.0760000000000001,"5min":0.94799999999999995,"15min":0.92600000000000005},"min":{"1min":0.56499999999999995,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.3380000000000001,"5min":1.3380000000000001,"15min":1.3380000000000001},"last":1.288},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.089,"15min":1.0780000000000001},"min":{"1min":0.59499999999999997,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.4670000000000001,"5min":1.5309999999999999,"15min":1.5309999999999999},"last":1.3109999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-13T07:31:45.800 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-13T07:31:45.800 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-13T07:31:45.800 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-13T07:31:45.800 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:31:46.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:45 smithi089 bash[21353]: cluster 2024-04-13T07:31:44.883171+0000 mgr.y (mgr.14373) 98 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:46.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:45 smithi089 bash[21353]: audit 2024-04-13T07:31:45.240192+0000 mgr.y (mgr.14373) 99 : audit 0 from='client.14553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:31:46.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:45 smithi089 bash[31522]: cluster 2024-04-13T07:31:44.883171+0000 mgr.y (mgr.14373) 98 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:46.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:45 smithi089 bash[31522]: audit 2024-04-13T07:31:45.240192+0000 mgr.y (mgr.14373) 99 : audit 0 from='client.14553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:31:46.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:45 smithi203 bash[23480]: cluster 2024-04-13T07:31:44.883171+0000 mgr.y (mgr.14373) 98 : cluster 0 pgmap v70: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:46.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:45 smithi203 bash[23480]: audit 2024-04-13T07:31:45.240192+0000 mgr.y (mgr.14373) 99 : audit 0 from='client.14553 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:31:46.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:31:46 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:31:46] "GET /metrics HTTP/1.1" 200 28557 "" "Prometheus/2.43.0" 2024-04-13T07:31:48.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:47 smithi203 bash[23480]: cluster 2024-04-13T07:31:46.883799+0000 mgr.y (mgr.14373) 100 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:48.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:47 smithi089 bash[21353]: cluster 2024-04-13T07:31:46.883799+0000 mgr.y (mgr.14373) 100 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:48.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:47 smithi089 bash[31522]: cluster 2024-04-13T07:31:46.883799+0000 mgr.y (mgr.14373) 100 : cluster 0 pgmap v71: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:50.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:49 smithi203 bash[23480]: cluster 2024-04-13T07:31:48.884402+0000 mgr.y (mgr.14373) 101 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:50.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:49 smithi089 bash[21353]: cluster 2024-04-13T07:31:48.884402+0000 mgr.y (mgr.14373) 101 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:50.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:49 smithi089 bash[31522]: cluster 2024-04-13T07:31:48.884402+0000 mgr.y (mgr.14373) 101 : cluster 0 pgmap v72: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:50.533 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:31:52.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:51 smithi203 bash[23480]: cluster 2024-04-13T07:31:50.885097+0000 mgr.y (mgr.14373) 102 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:52.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:51 smithi089 bash[21353]: cluster 2024-04-13T07:31:50.885097+0000 mgr.y (mgr.14373) 102 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:52.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:51 smithi089 bash[31522]: cluster 2024-04-13T07:31:50.885097+0000 mgr.y (mgr.14373) 102 : cluster 0 pgmap v73: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:53.218 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:31:53.218 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-13T07:31:53.804 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-13T07:31:53.804 INFO:tasks.cephadm:Setup complete, yielding 2024-04-13T07:31:53.805 INFO:teuthology.run_tasks:Running task cephadm.shell... 2024-04-13T07:31:53.815 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi089.front.sepia.ceph.com 2024-04-13T07:31:53.815 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- bash -c 'ceph fs volume create foo' 2024-04-13T07:31:54.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:53 smithi203 bash[23480]: cluster 2024-04-13T07:31:52.885490+0000 mgr.y (mgr.14373) 103 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:54.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:53 smithi203 bash[23480]: audit 2024-04-13T07:31:53.225382+0000 mon.a (mon.0) 784 : audit 0 from='client.? 172.21.15.89:0/3545881542' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:31:54.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:53 smithi203 bash[23480]: audit 2024-04-13T07:31:53.931150+0000 mon.a (mon.0) 785 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:54.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:53 smithi089 bash[31522]: cluster 2024-04-13T07:31:52.885490+0000 mgr.y (mgr.14373) 103 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:54.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:53 smithi089 bash[31522]: audit 2024-04-13T07:31:53.225382+0000 mon.a (mon.0) 784 : audit 0 from='client.? 172.21.15.89:0/3545881542' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:31:54.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:53 smithi089 bash[31522]: audit 2024-04-13T07:31:53.931150+0000 mon.a (mon.0) 785 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:54.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:53 smithi089 bash[21353]: cluster 2024-04-13T07:31:52.885490+0000 mgr.y (mgr.14373) 103 : cluster 0 pgmap v74: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:54.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:53 smithi089 bash[21353]: audit 2024-04-13T07:31:53.225382+0000 mon.a (mon.0) 784 : audit 0 from='client.? 172.21.15.89:0/3545881542' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:31:54.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:53 smithi089 bash[21353]: audit 2024-04-13T07:31:53.931150+0000 mon.a (mon.0) 785 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:31:56.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:55 smithi203 bash[23480]: cluster 2024-04-13T07:31:54.886013+0000 mgr.y (mgr.14373) 104 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:56.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:31:56 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:31:56] "GET /metrics HTTP/1.1" 200 28567 "" "Prometheus/2.43.0" 2024-04-13T07:31:56.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:55 smithi089 bash[31522]: cluster 2024-04-13T07:31:54.886013+0000 mgr.y (mgr.14373) 104 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:56.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:55 smithi089 bash[21353]: cluster 2024-04-13T07:31:54.886013+0000 mgr.y (mgr.14373) 104 : cluster 0 pgmap v75: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:57.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:56 smithi203 bash[23480]: audit 2024-04-13T07:31:56.376016+0000 mon.a (mon.0) 786 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:31:57.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:56 smithi203 bash[23480]: audit 2024-04-13T07:31:56.807911+0000 mon.a (mon.0) 787 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:57.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:56 smithi203 bash[23480]: audit 2024-04-13T07:31:56.818600+0000 mon.a (mon.0) 788 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:57.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:56 smithi089 bash[31522]: audit 2024-04-13T07:31:56.376016+0000 mon.a (mon.0) 786 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:31:57.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:56 smithi089 bash[31522]: audit 2024-04-13T07:31:56.807911+0000 mon.a (mon.0) 787 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:57.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:56 smithi089 bash[31522]: audit 2024-04-13T07:31:56.818600+0000 mon.a (mon.0) 788 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:57.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:56 smithi089 bash[21353]: audit 2024-04-13T07:31:56.376016+0000 mon.a (mon.0) 786 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:31:57.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:56 smithi089 bash[21353]: audit 2024-04-13T07:31:56.807911+0000 mon.a (mon.0) 787 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:57.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:56 smithi089 bash[21353]: audit 2024-04-13T07:31:56.818600+0000 mon.a (mon.0) 788 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:57 smithi203 bash[23480]: cluster 2024-04-13T07:31:56.886680+0000 mgr.y (mgr.14373) 105 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:57 smithi203 bash[23480]: audit 2024-04-13T07:31:57.498158+0000 mon.a (mon.0) 789 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:31:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:57 smithi203 bash[23480]: audit 2024-04-13T07:31:57.500056+0000 mon.a (mon.0) 790 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:31:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:31:57 smithi203 bash[23480]: audit 2024-04-13T07:31:57.510676+0000 mon.a (mon.0) 791 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:58.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:58 smithi089 bash[31522]: cluster 2024-04-13T07:31:56.886680+0000 mgr.y (mgr.14373) 105 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:58.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:58 smithi089 bash[31522]: audit 2024-04-13T07:31:57.498158+0000 mon.a (mon.0) 789 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:31:58.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:58 smithi089 bash[31522]: audit 2024-04-13T07:31:57.500056+0000 mon.a (mon.0) 790 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:31:58.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:31:58 smithi089 bash[31522]: audit 2024-04-13T07:31:57.510676+0000 mon.a (mon.0) 791 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:58.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:57 smithi089 bash[21353]: cluster 2024-04-13T07:31:56.886680+0000 mgr.y (mgr.14373) 105 : cluster 0 pgmap v76: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:31:58.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:57 smithi089 bash[21353]: audit 2024-04-13T07:31:57.498158+0000 mon.a (mon.0) 789 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:31:58.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:57 smithi089 bash[21353]: audit 2024-04-13T07:31:57.500056+0000 mon.a (mon.0) 790 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:31:58.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:31:58 smithi089 bash[21353]: audit 2024-04-13T07:31:57.510676+0000 mon.a (mon.0) 791 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:31:58.545 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:00.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:00 smithi203 bash[23480]: cluster 2024-04-13T07:31:58.887274+0000 mgr.y (mgr.14373) 106 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:00.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:00 smithi089 bash[21353]: cluster 2024-04-13T07:31:58.887274+0000 mgr.y (mgr.14373) 106 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:00.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:00 smithi089 bash[31522]: cluster 2024-04-13T07:31:58.887274+0000 mgr.y (mgr.14373) 106 : cluster 0 pgmap v77: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:02.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:02 smithi203 bash[23480]: cluster 2024-04-13T07:32:00.887713+0000 mgr.y (mgr.14373) 107 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:02.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:02 smithi203 bash[23480]: audit 2024-04-13T07:32:01.126810+0000 mgr.y (mgr.14373) 108 : audit 0 from='client.24467 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:32:02.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:02 smithi203 bash[23480]: audit 2024-04-13T07:32:01.127780+0000 mon.a (mon.0) 792 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-13T07:32:02.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:02 smithi089 bash[21353]: cluster 2024-04-13T07:32:00.887713+0000 mgr.y (mgr.14373) 107 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:02.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:02 smithi089 bash[21353]: audit 2024-04-13T07:32:01.126810+0000 mgr.y (mgr.14373) 108 : audit 0 from='client.24467 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:32:02.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:02 smithi089 bash[21353]: audit 2024-04-13T07:32:01.127780+0000 mon.a (mon.0) 792 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-13T07:32:02.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:02 smithi089 bash[31522]: cluster 2024-04-13T07:32:00.887713+0000 mgr.y (mgr.14373) 107 : cluster 0 pgmap v78: 9 pgs: 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:02.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:02 smithi089 bash[31522]: audit 2024-04-13T07:32:01.126810+0000 mgr.y (mgr.14373) 108 : audit 0 from='client.24467 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:32:02.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:02 smithi089 bash[31522]: audit 2024-04-13T07:32:01.127780+0000 mon.a (mon.0) 792 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2024-04-13T07:32:03.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:03 smithi089 bash[31522]: audit 2024-04-13T07:32:02.022781+0000 mon.a (mon.0) 793 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-13T07:32:03.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:03 smithi089 bash[31522]: cluster 2024-04-13T07:32:02.022970+0000 mon.a (mon.0) 794 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-04-13T07:32:03.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:03 smithi089 bash[31522]: audit 2024-04-13T07:32:02.028178+0000 mon.a (mon.0) 795 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-13T07:32:03.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:03 smithi089 bash[21353]: audit 2024-04-13T07:32:02.022781+0000 mon.a (mon.0) 793 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-13T07:32:03.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:03 smithi089 bash[21353]: cluster 2024-04-13T07:32:02.022970+0000 mon.a (mon.0) 794 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-04-13T07:32:03.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:03 smithi089 bash[21353]: audit 2024-04-13T07:32:02.028178+0000 mon.a (mon.0) 795 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-13T07:32:03.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:03 smithi089 bash[21353]: debug 2024-04-13T07:32:03.050+0000 7f144a846700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-13T07:32:03.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:03 smithi203 bash[23480]: audit 2024-04-13T07:32:02.022781+0000 mon.a (mon.0) 793 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2024-04-13T07:32:03.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:03 smithi203 bash[23480]: cluster 2024-04-13T07:32:02.022970+0000 mon.a (mon.0) 794 : cluster 0 osdmap e65: 8 total, 8 up, 8 in 2024-04-13T07:32:03.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:03 smithi203 bash[23480]: audit 2024-04-13T07:32:02.028178+0000 mon.a (mon.0) 795 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2024-04-13T07:32:04.161 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2024-04-13T07:32:04.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cluster 2024-04-13T07:32:02.888340+0000 mgr.y (mgr.14373) 109 : cluster 0 pgmap v80: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cluster 2024-04-13T07:32:03.015418+0000 mon.a (mon.0) 796 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.037998+0000 mon.a (mon.0) 797 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cluster 2024-04-13T07:32:03.038357+0000 mon.a (mon.0) 798 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.052497+0000 mon.a (mon.0) 799 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cluster 2024-04-13T07:32:03.054559+0000 mon.a (mon.0) 800 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cluster 2024-04-13T07:32:03.054628+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cluster 2024-04-13T07:32:03.081717+0000 mon.a (mon.0) 802 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.082976+0000 mon.a (mon.0) 803 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cluster 2024-04-13T07:32:03.083191+0000 mon.a (mon.0) 804 : cluster 0 fsmap foo:0 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cephadm 2024-04-13T07:32:03.090420+0000 mgr.y (mgr.14373) 110 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.098847+0000 mon.a (mon.0) 805 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:04.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.126438+0000 mon.a (mon.0) 806 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.128132+0000 mon.a (mon.0) 807 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.128863+0000 mon.a (mon.0) 808 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.134440+0000 mon.a (mon.0) 809 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.140689+0000 mon.a (mon.0) 810 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.147955+0000 mon.a (mon.0) 811 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: audit 2024-04-13T07:32:03.149125+0000 mon.a (mon.0) 812 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[21353]: cephadm 2024-04-13T07:32:03.150200+0000 mgr.y (mgr.14373) 111 : cephadm 1 Deploying daemon mds.foo.smithi089.pccnvq on smithi089 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cluster 2024-04-13T07:32:02.888340+0000 mgr.y (mgr.14373) 109 : cluster 0 pgmap v80: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cluster 2024-04-13T07:32:03.015418+0000 mon.a (mon.0) 796 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.037998+0000 mon.a (mon.0) 797 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-13T07:32:04.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cluster 2024-04-13T07:32:03.038357+0000 mon.a (mon.0) 798 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.052497+0000 mon.a (mon.0) 799 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cluster 2024-04-13T07:32:03.054559+0000 mon.a (mon.0) 800 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cluster 2024-04-13T07:32:03.054628+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cluster 2024-04-13T07:32:03.081717+0000 mon.a (mon.0) 802 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.082976+0000 mon.a (mon.0) 803 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cluster 2024-04-13T07:32:03.083191+0000 mon.a (mon.0) 804 : cluster 0 fsmap foo:0 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cephadm 2024-04-13T07:32:03.090420+0000 mgr.y (mgr.14373) 110 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.098847+0000 mon.a (mon.0) 805 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.126438+0000 mon.a (mon.0) 806 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.128132+0000 mon.a (mon.0) 807 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.128863+0000 mon.a (mon.0) 808 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.134440+0000 mon.a (mon.0) 809 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.140689+0000 mon.a (mon.0) 810 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:32:04.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.147955+0000 mon.a (mon.0) 811 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-13T07:32:04.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: audit 2024-04-13T07:32:03.149125+0000 mon.a (mon.0) 812 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:04.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:04 smithi089 bash[31522]: cephadm 2024-04-13T07:32:03.150200+0000 mgr.y (mgr.14373) 111 : cephadm 1 Deploying daemon mds.foo.smithi089.pccnvq on smithi089 2024-04-13T07:32:04.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cluster 2024-04-13T07:32:02.888340+0000 mgr.y (mgr.14373) 109 : cluster 0 pgmap v80: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:04.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cluster 2024-04-13T07:32:03.015418+0000 mon.a (mon.0) 796 : cluster 3 Health check failed: 1 pool(s) do not have an application enabled (POOL_APP_NOT_ENABLED) 2024-04-13T07:32:04.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.037998+0000 mon.a (mon.0) 797 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"bulk": true, "prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cluster 2024-04-13T07:32:03.038357+0000 mon.a (mon.0) 798 : cluster 0 osdmap e66: 8 total, 8 up, 8 in 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.052497+0000 mon.a (mon.0) 799 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cluster 2024-04-13T07:32:03.054559+0000 mon.a (mon.0) 800 : cluster 4 Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cluster 2024-04-13T07:32:03.054628+0000 mon.a (mon.0) 801 : cluster 3 Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cluster 2024-04-13T07:32:03.081717+0000 mon.a (mon.0) 802 : cluster 0 osdmap e67: 8 total, 8 up, 8 in 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.082976+0000 mon.a (mon.0) 803 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cluster 2024-04-13T07:32:03.083191+0000 mon.a (mon.0) 804 : cluster 0 fsmap foo:0 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cephadm 2024-04-13T07:32:03.090420+0000 mgr.y (mgr.14373) 110 : cephadm 1 Saving service mds.foo spec with placement count:2 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.098847+0000 mon.a (mon.0) 805 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.126438+0000 mon.a (mon.0) 806 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.128132+0000 mon.a (mon.0) 807 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.128863+0000 mon.a (mon.0) 808 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.134440+0000 mon.a (mon.0) 809 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.140689+0000 mon.a (mon.0) 810 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:32:04.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.147955+0000 mon.a (mon.0) 811 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-13T07:32:04.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: audit 2024-04-13T07:32:03.149125+0000 mon.a (mon.0) 812 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:04.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:04 smithi203 bash[23480]: cephadm 2024-04-13T07:32:03.150200+0000 mgr.y (mgr.14373) 111 : cephadm 1 Deploying daemon mds.foo.smithi089.pccnvq on smithi089 2024-04-13T07:32:05.431 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:05 smithi089 bash[31522]: cluster 2024-04-13T07:32:04.083411+0000 mon.a (mon.0) 813 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-04-13T07:32:05.431 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:05 smithi089 bash[21353]: cluster 2024-04-13T07:32:04.083411+0000 mon.a (mon.0) 813 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-04-13T07:32:05.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:05 smithi203 bash[23480]: cluster 2024-04-13T07:32:04.083411+0000 mon.a (mon.0) 813 : cluster 0 osdmap e68: 8 total, 8 up, 8 in 2024-04-13T07:32:05.695 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:05.695 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:05.695 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:05.696 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:05.696 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:05.696 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:05.697 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:05.697 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:05.697 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.048 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.048 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.049 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.049 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.049 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:32:05 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: cluster 2024-04-13T07:32:04.889098+0000 mgr.y (mgr.14373) 112 : cluster 0 pgmap v84: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: cluster 2024-04-13T07:32:05.074937+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-13T07:32:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: cluster 2024-04-13T07:32:05.092576+0000 mon.a (mon.0) 815 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-04-13T07:32:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: audit 2024-04-13T07:32:05.857984+0000 mon.a (mon.0) 816 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: audit 2024-04-13T07:32:05.864039+0000 mon.a (mon.0) 817 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: audit 2024-04-13T07:32:05.873343+0000 mon.a (mon.0) 818 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: audit 2024-04-13T07:32:05.873961+0000 mon.a (mon.0) 819 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:32:06.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: audit 2024-04-13T07:32:05.878156+0000 mon.a (mon.0) 820 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-13T07:32:06.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:06 smithi203 bash[23480]: audit 2024-04-13T07:32:05.878886+0000 mon.a (mon.0) 821 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:06.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:32:06] "GET /metrics HTTP/1.1" 200 28567 "" "Prometheus/2.43.0" 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: cluster 2024-04-13T07:32:04.889098+0000 mgr.y (mgr.14373) 112 : cluster 0 pgmap v84: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: cluster 2024-04-13T07:32:05.074937+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: cluster 2024-04-13T07:32:05.092576+0000 mon.a (mon.0) 815 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: audit 2024-04-13T07:32:05.857984+0000 mon.a (mon.0) 816 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: audit 2024-04-13T07:32:05.864039+0000 mon.a (mon.0) 817 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: audit 2024-04-13T07:32:05.873343+0000 mon.a (mon.0) 818 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: audit 2024-04-13T07:32:05.873961+0000 mon.a (mon.0) 819 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: audit 2024-04-13T07:32:05.878156+0000 mon.a (mon.0) 820 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-13T07:32:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[31522]: audit 2024-04-13T07:32:05.878886+0000 mon.a (mon.0) 821 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: cluster 2024-04-13T07:32:04.889098+0000 mgr.y (mgr.14373) 112 : cluster 0 pgmap v84: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: cluster 2024-04-13T07:32:05.074937+0000 mon.a (mon.0) 814 : cluster 1 Health check cleared: POOL_APP_NOT_ENABLED (was: 1 pool(s) do not have an application enabled) 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: cluster 2024-04-13T07:32:05.092576+0000 mon.a (mon.0) 815 : cluster 0 osdmap e69: 8 total, 8 up, 8 in 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: audit 2024-04-13T07:32:05.857984+0000 mon.a (mon.0) 816 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: audit 2024-04-13T07:32:05.864039+0000 mon.a (mon.0) 817 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: audit 2024-04-13T07:32:05.873343+0000 mon.a (mon.0) 818 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: audit 2024-04-13T07:32:05.873961+0000 mon.a (mon.0) 819 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: audit 2024-04-13T07:32:05.878156+0000 mon.a (mon.0) 820 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2024-04-13T07:32:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:06 smithi089 bash[21353]: audit 2024-04-13T07:32:05.878886+0000 mon.a (mon.0) 821 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:07.348 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:07 smithi203 bash[23480]: cephadm 2024-04-13T07:32:05.879822+0000 mgr.y (mgr.14373) 113 : cephadm 1 Deploying daemon mds.foo.smithi203.sjspdy on smithi203 2024-04-13T07:32:07.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:07 smithi089 bash[21353]: cephadm 2024-04-13T07:32:05.879822+0000 mgr.y (mgr.14373) 113 : cephadm 1 Deploying daemon mds.foo.smithi203.sjspdy on smithi203 2024-04-13T07:32:07.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:07 smithi089 bash[31522]: cephadm 2024-04-13T07:32:05.879822+0000 mgr.y (mgr.14373) 113 : cephadm 1 Deploying daemon mds.foo.smithi203.sjspdy on smithi203 2024-04-13T07:32:08.422 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.422 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.422 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.423 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.423 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.423 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.423 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.424 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.424 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:06.889505+0000 mgr.y (mgr.14373) 114 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:08.424 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:07.099848+0000 mon.a (mon.0) 822 : cluster 0 mds.? [v2:172.21.15.89:6834/611307576,v1:172.21.15.89:6835/611307576] up:boot 2024-04-13T07:32:08.424 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:07.099918+0000 mon.a (mon.0) 823 : cluster 1 daemon mds.foo.smithi089.pccnvq assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-13T07:32:08.424 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:07.099986+0000 mon.a (mon.0) 824 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-13T07:32:08.424 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:07.100018+0000 mon.a (mon.0) 825 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-13T07:32:08.424 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:07.100041+0000 mon.a (mon.0) 826 : cluster 1 Cluster is now healthy 2024-04-13T07:32:08.424 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:07.100360+0000 mon.a (mon.0) 827 : cluster 0 fsmap foo:0 1 up:standby 2024-04-13T07:32:08.425 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: audit 2024-04-13T07:32:07.100549+0000 mon.a (mon.0) 828 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:32:08.425 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:07.106115+0000 mon.a (mon.0) 829 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:creating} 2024-04-13T07:32:08.425 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 bash[23480]: cluster 2024-04-13T07:32:07.131064+0000 mon.a (mon.0) 830 : cluster 1 daemon mds.foo.smithi089.pccnvq is now active in filesystem foo as rank 0 2024-04-13T07:32:08.425 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:06.889505+0000 mgr.y (mgr.14373) 114 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:07.099848+0000 mon.a (mon.0) 822 : cluster 0 mds.? [v2:172.21.15.89:6834/611307576,v1:172.21.15.89:6835/611307576] up:boot 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:07.099918+0000 mon.a (mon.0) 823 : cluster 1 daemon mds.foo.smithi089.pccnvq assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:07.099986+0000 mon.a (mon.0) 824 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:07.100018+0000 mon.a (mon.0) 825 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:07.100041+0000 mon.a (mon.0) 826 : cluster 1 Cluster is now healthy 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:07.100360+0000 mon.a (mon.0) 827 : cluster 0 fsmap foo:0 1 up:standby 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: audit 2024-04-13T07:32:07.100549+0000 mon.a (mon.0) 828 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:07.106115+0000 mon.a (mon.0) 829 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:creating} 2024-04-13T07:32:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[21353]: cluster 2024-04-13T07:32:07.131064+0000 mon.a (mon.0) 830 : cluster 1 daemon mds.foo.smithi089.pccnvq is now active in filesystem foo as rank 0 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:06.889505+0000 mgr.y (mgr.14373) 114 : cluster 0 pgmap v86: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:07.099848+0000 mon.a (mon.0) 822 : cluster 0 mds.? [v2:172.21.15.89:6834/611307576,v1:172.21.15.89:6835/611307576] up:boot 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:07.099918+0000 mon.a (mon.0) 823 : cluster 1 daemon mds.foo.smithi089.pccnvq assigned to filesystem foo as rank 0 (now has 1 ranks) 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:07.099986+0000 mon.a (mon.0) 824 : cluster 1 Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:07.100018+0000 mon.a (mon.0) 825 : cluster 1 Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:07.100041+0000 mon.a (mon.0) 826 : cluster 1 Cluster is now healthy 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:07.100360+0000 mon.a (mon.0) 827 : cluster 0 fsmap foo:0 1 up:standby 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: audit 2024-04-13T07:32:07.100549+0000 mon.a (mon.0) 828 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:07.106115+0000 mon.a (mon.0) 829 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:creating} 2024-04-13T07:32:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:08 smithi089 bash[31522]: cluster 2024-04-13T07:32:07.131064+0000 mon.a (mon.0) 830 : cluster 1 daemon mds.foo.smithi089.pccnvq is now active in filesystem foo as rank 0 2024-04-13T07:32:08.689 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.690 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.690 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.691 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.691 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.691 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.692 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.692 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.692 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:32:08 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:32:08.896 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:09.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: cluster 2024-04-13T07:32:08.111159+0000 mon.a (mon.0) 831 : cluster 0 mds.? [v2:172.21.15.89:6834/611307576,v1:172.21.15.89:6835/611307576] up:active 2024-04-13T07:32:09.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: cluster 2024-04-13T07:32:08.111299+0000 mon.a (mon.0) 832 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:active} 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: audit 2024-04-13T07:32:08.739786+0000 mon.a (mon.0) 833 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: audit 2024-04-13T07:32:08.751290+0000 mon.a (mon.0) 834 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: audit 2024-04-13T07:32:08.808379+0000 mon.a (mon.0) 835 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: audit 2024-04-13T07:32:08.818060+0000 mon.a (mon.0) 836 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: audit 2024-04-13T07:32:08.828340+0000 mon.a (mon.0) 837 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: audit 2024-04-13T07:32:08.895657+0000 mon.a (mon.0) 838 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[21353]: audit 2024-04-13T07:32:08.931360+0000 mon.a (mon.0) 839 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: cluster 2024-04-13T07:32:08.111159+0000 mon.a (mon.0) 831 : cluster 0 mds.? [v2:172.21.15.89:6834/611307576,v1:172.21.15.89:6835/611307576] up:active 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: cluster 2024-04-13T07:32:08.111299+0000 mon.a (mon.0) 832 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:active} 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: audit 2024-04-13T07:32:08.739786+0000 mon.a (mon.0) 833 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: audit 2024-04-13T07:32:08.751290+0000 mon.a (mon.0) 834 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: audit 2024-04-13T07:32:08.808379+0000 mon.a (mon.0) 835 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: audit 2024-04-13T07:32:08.818060+0000 mon.a (mon.0) 836 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: audit 2024-04-13T07:32:08.828340+0000 mon.a (mon.0) 837 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: audit 2024-04-13T07:32:08.895657+0000 mon.a (mon.0) 838 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:09.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:09 smithi089 bash[31522]: audit 2024-04-13T07:32:08.931360+0000 mon.a (mon.0) 839 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: cluster 2024-04-13T07:32:08.111159+0000 mon.a (mon.0) 831 : cluster 0 mds.? [v2:172.21.15.89:6834/611307576,v1:172.21.15.89:6835/611307576] up:active 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: cluster 2024-04-13T07:32:08.111299+0000 mon.a (mon.0) 832 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:active} 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: audit 2024-04-13T07:32:08.739786+0000 mon.a (mon.0) 833 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: audit 2024-04-13T07:32:08.751290+0000 mon.a (mon.0) 834 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: audit 2024-04-13T07:32:08.808379+0000 mon.a (mon.0) 835 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: audit 2024-04-13T07:32:08.818060+0000 mon.a (mon.0) 836 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: audit 2024-04-13T07:32:08.828340+0000 mon.a (mon.0) 837 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: audit 2024-04-13T07:32:08.895657+0000 mon.a (mon.0) 838 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:09.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:09 smithi203 bash[23480]: audit 2024-04-13T07:32:08.931360+0000 mon.a (mon.0) 839 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:10.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:10 smithi089 bash[31522]: cluster 2024-04-13T07:32:08.890064+0000 mgr.y (mgr.14373) 115 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:10.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:10 smithi089 bash[31522]: audit 2024-04-13T07:32:09.287767+0000 mon.a (mon.0) 840 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:10.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:10 smithi089 bash[21353]: cluster 2024-04-13T07:32:08.890064+0000 mgr.y (mgr.14373) 115 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:10.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:10 smithi089 bash[21353]: audit 2024-04-13T07:32:09.287767+0000 mon.a (mon.0) 840 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:10.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:10 smithi203 bash[23480]: cluster 2024-04-13T07:32:08.890064+0000 mgr.y (mgr.14373) 115 : cluster 0 pgmap v87: 73 pgs: 73 active+clean; 577 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:10.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:10 smithi203 bash[23480]: audit 2024-04-13T07:32:09.287767+0000 mon.a (mon.0) 840 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.834876+0000 mon.a (mon.0) 841 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:boot 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.834969+0000 mon.a (mon.0) 842 : cluster 1 Dropping low affinity active daemon mds.foo.smithi089.pccnvq in favor of higher affinity standby. 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.835064+0000 mon.a (mon.0) 843 : cluster 3 Replacing daemon mds.foo.smithi089.pccnvq as rank 0 with standby daemon mds.foo.smithi203.sjspdy 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.836089+0000 mon.a (mon.0) 844 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.836157+0000 mon.a (mon.0) 845 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.836242+0000 mon.a (mon.0) 846 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:active} 1 up:standby 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: audit 2024-04-13T07:32:10.836492+0000 mon.a (mon.0) 847 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.845247+0000 mon.a (mon.0) 848 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.853106+0000 mon.a (mon.0) 849 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:replay} 2024-04-13T07:32:12.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:11 smithi203 bash[23480]: cluster 2024-04-13T07:32:10.890504+0000 mgr.y (mgr.14373) 116 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s wr, 6 op/s 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.834876+0000 mon.a (mon.0) 841 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:boot 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.834969+0000 mon.a (mon.0) 842 : cluster 1 Dropping low affinity active daemon mds.foo.smithi089.pccnvq in favor of higher affinity standby. 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.835064+0000 mon.a (mon.0) 843 : cluster 3 Replacing daemon mds.foo.smithi089.pccnvq as rank 0 with standby daemon mds.foo.smithi203.sjspdy 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.836089+0000 mon.a (mon.0) 844 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.836157+0000 mon.a (mon.0) 845 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.836242+0000 mon.a (mon.0) 846 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:active} 1 up:standby 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: audit 2024-04-13T07:32:10.836492+0000 mon.a (mon.0) 847 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.845247+0000 mon.a (mon.0) 848 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.853106+0000 mon.a (mon.0) 849 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:replay} 2024-04-13T07:32:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[21353]: cluster 2024-04-13T07:32:10.890504+0000 mgr.y (mgr.14373) 116 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s wr, 6 op/s 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.834876+0000 mon.a (mon.0) 841 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:boot 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.834969+0000 mon.a (mon.0) 842 : cluster 1 Dropping low affinity active daemon mds.foo.smithi089.pccnvq in favor of higher affinity standby. 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.835064+0000 mon.a (mon.0) 843 : cluster 3 Replacing daemon mds.foo.smithi089.pccnvq as rank 0 with standby daemon mds.foo.smithi203.sjspdy 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.836089+0000 mon.a (mon.0) 844 : cluster 3 Health check failed: 1 filesystem is degraded (FS_DEGRADED) 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.836157+0000 mon.a (mon.0) 845 : cluster 3 Health check failed: insufficient standby MDS daemons available (MDS_INSUFFICIENT_STANDBY) 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.836242+0000 mon.a (mon.0) 846 : cluster 0 fsmap foo:1 {0=foo.smithi089.pccnvq=up:active} 1 up:standby 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: audit 2024-04-13T07:32:10.836492+0000 mon.a (mon.0) 847 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.845247+0000 mon.a (mon.0) 848 : cluster 0 osdmap e70: 8 total, 8 up, 8 in 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.853106+0000 mon.a (mon.0) 849 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:replay} 2024-04-13T07:32:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:11 smithi089 bash[31522]: cluster 2024-04-13T07:32:10.890504+0000 mgr.y (mgr.14373) 116 : cluster 0 pgmap v89: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 2.1 KiB/s wr, 6 op/s 2024-04-13T07:32:12.541 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2024-04-13T07:32:13.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:12 smithi203 bash[23480]: cluster 2024-04-13T07:32:11.847713+0000 mon.a (mon.0) 850 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-13T07:32:13.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:12 smithi203 bash[23480]: cluster 2024-04-13T07:32:11.857374+0000 mon.a (mon.0) 851 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:reconnect 2024-04-13T07:32:13.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:12 smithi203 bash[23480]: cluster 2024-04-13T07:32:11.857414+0000 mon.a (mon.0) 852 : cluster 0 mds.? [v2:172.21.15.89:6834/2537258916,v1:172.21.15.89:6835/2537258916] up:boot 2024-04-13T07:32:13.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:12 smithi203 bash[23480]: cluster 2024-04-13T07:32:11.857493+0000 mon.a (mon.0) 853 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:reconnect} 1 up:standby 2024-04-13T07:32:13.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:12 smithi203 bash[23480]: audit 2024-04-13T07:32:11.857599+0000 mon.a (mon.0) 854 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:32:13.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[31522]: cluster 2024-04-13T07:32:11.847713+0000 mon.a (mon.0) 850 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-13T07:32:13.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[31522]: cluster 2024-04-13T07:32:11.857374+0000 mon.a (mon.0) 851 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:reconnect 2024-04-13T07:32:13.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[31522]: cluster 2024-04-13T07:32:11.857414+0000 mon.a (mon.0) 852 : cluster 0 mds.? [v2:172.21.15.89:6834/2537258916,v1:172.21.15.89:6835/2537258916] up:boot 2024-04-13T07:32:13.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[31522]: cluster 2024-04-13T07:32:11.857493+0000 mon.a (mon.0) 853 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:reconnect} 1 up:standby 2024-04-13T07:32:13.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[31522]: audit 2024-04-13T07:32:11.857599+0000 mon.a (mon.0) 854 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:32:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[21353]: cluster 2024-04-13T07:32:11.847713+0000 mon.a (mon.0) 850 : cluster 1 Health check cleared: MDS_INSUFFICIENT_STANDBY (was: insufficient standby MDS daemons available) 2024-04-13T07:32:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[21353]: cluster 2024-04-13T07:32:11.857374+0000 mon.a (mon.0) 851 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:reconnect 2024-04-13T07:32:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[21353]: cluster 2024-04-13T07:32:11.857414+0000 mon.a (mon.0) 852 : cluster 0 mds.? [v2:172.21.15.89:6834/2537258916,v1:172.21.15.89:6835/2537258916] up:boot 2024-04-13T07:32:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[21353]: cluster 2024-04-13T07:32:11.857493+0000 mon.a (mon.0) 853 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:reconnect} 1 up:standby 2024-04-13T07:32:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:12 smithi089 bash[21353]: audit 2024-04-13T07:32:11.857599+0000 mon.a (mon.0) 854 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:32:14.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:13 smithi203 bash[23480]: cluster 2024-04-13T07:32:12.860154+0000 mon.a (mon.0) 855 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:rejoin 2024-04-13T07:32:14.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:13 smithi203 bash[23480]: cluster 2024-04-13T07:32:12.860265+0000 mon.a (mon.0) 856 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:rejoin} 1 up:standby 2024-04-13T07:32:14.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:13 smithi203 bash[23480]: cluster 2024-04-13T07:32:12.860954+0000 mon.a (mon.0) 857 : cluster 1 daemon mds.foo.smithi203.sjspdy is now active in filesystem foo as rank 0 2024-04-13T07:32:14.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:13 smithi203 bash[23480]: cluster 2024-04-13T07:32:12.891124+0000 mgr.y (mgr.14373) 117 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-04-13T07:32:14.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:13 smithi203 bash[23480]: audit 2024-04-13T07:32:13.632766+0000 mon.a (mon.0) 858 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:14.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:13 smithi203 bash[23480]: audit 2024-04-13T07:32:13.640922+0000 mon.a (mon.0) 859 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:14.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[21353]: cluster 2024-04-13T07:32:12.860154+0000 mon.a (mon.0) 855 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:rejoin 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[21353]: cluster 2024-04-13T07:32:12.860265+0000 mon.a (mon.0) 856 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:rejoin} 1 up:standby 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[21353]: cluster 2024-04-13T07:32:12.860954+0000 mon.a (mon.0) 857 : cluster 1 daemon mds.foo.smithi203.sjspdy is now active in filesystem foo as rank 0 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[21353]: cluster 2024-04-13T07:32:12.891124+0000 mgr.y (mgr.14373) 117 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[21353]: audit 2024-04-13T07:32:13.632766+0000 mon.a (mon.0) 858 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[21353]: audit 2024-04-13T07:32:13.640922+0000 mon.a (mon.0) 859 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[31522]: cluster 2024-04-13T07:32:12.860154+0000 mon.a (mon.0) 855 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:rejoin 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[31522]: cluster 2024-04-13T07:32:12.860265+0000 mon.a (mon.0) 856 : cluster 0 fsmap foo:1/1 {0=foo.smithi203.sjspdy=up:rejoin} 1 up:standby 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[31522]: cluster 2024-04-13T07:32:12.860954+0000 mon.a (mon.0) 857 : cluster 1 daemon mds.foo.smithi203.sjspdy is now active in filesystem foo as rank 0 2024-04-13T07:32:14.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[31522]: cluster 2024-04-13T07:32:12.891124+0000 mgr.y (mgr.14373) 117 : cluster 0 pgmap v90: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 1.7 KiB/s wr, 5 op/s 2024-04-13T07:32:14.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[31522]: audit 2024-04-13T07:32:13.632766+0000 mon.a (mon.0) 858 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:14.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:13 smithi089 bash[31522]: audit 2024-04-13T07:32:13.640922+0000 mon.a (mon.0) 859 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: cluster 2024-04-13T07:32:13.856376+0000 mon.a (mon.0) 860 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-13T07:32:15.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: cluster 2024-04-13T07:32:13.856428+0000 mon.a (mon.0) 861 : cluster 1 Cluster is now healthy 2024-04-13T07:32:15.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: cluster 2024-04-13T07:32:13.871221+0000 mon.a (mon.0) 862 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:active 2024-04-13T07:32:15.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: cluster 2024-04-13T07:32:13.871401+0000 mon.a (mon.0) 863 : cluster 0 fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:32:15.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.297845+0000 mon.a (mon.0) 864 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.641634+0000 mon.a (mon.0) 865 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.653468+0000 mon.a (mon.0) 866 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.656210+0000 mon.a (mon.0) 867 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:15.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.657984+0000 mon.a (mon.0) 868 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:15.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.668490+0000 mon.a (mon.0) 869 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.760244+0000 mon.a (mon.0) 870 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:15.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.763218+0000 mon.a (mon.0) 871 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:15.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.764911+0000 mon.a (mon.0) 872 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:15.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:14 smithi203 bash[23480]: audit 2024-04-13T07:32:14.774856+0000 mon.a (mon.0) 873 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: cluster 2024-04-13T07:32:13.856376+0000 mon.a (mon.0) 860 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: cluster 2024-04-13T07:32:13.856428+0000 mon.a (mon.0) 861 : cluster 1 Cluster is now healthy 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: cluster 2024-04-13T07:32:13.871221+0000 mon.a (mon.0) 862 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:active 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: cluster 2024-04-13T07:32:13.871401+0000 mon.a (mon.0) 863 : cluster 0 fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.297845+0000 mon.a (mon.0) 864 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.641634+0000 mon.a (mon.0) 865 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.653468+0000 mon.a (mon.0) 866 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.656210+0000 mon.a (mon.0) 867 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.657984+0000 mon.a (mon.0) 868 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.668490+0000 mon.a (mon.0) 869 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.760244+0000 mon.a (mon.0) 870 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.763218+0000 mon.a (mon.0) 871 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.764911+0000 mon.a (mon.0) 872 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[21353]: audit 2024-04-13T07:32:14.774856+0000 mon.a (mon.0) 873 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: cluster 2024-04-13T07:32:13.856376+0000 mon.a (mon.0) 860 : cluster 1 Health check cleared: FS_DEGRADED (was: 1 filesystem is degraded) 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: cluster 2024-04-13T07:32:13.856428+0000 mon.a (mon.0) 861 : cluster 1 Cluster is now healthy 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: cluster 2024-04-13T07:32:13.871221+0000 mon.a (mon.0) 862 : cluster 0 mds.? [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] up:active 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: cluster 2024-04-13T07:32:13.871401+0000 mon.a (mon.0) 863 : cluster 0 fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.297845+0000 mon.a (mon.0) 864 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.641634+0000 mon.a (mon.0) 865 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.653468+0000 mon.a (mon.0) 866 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.656210+0000 mon.a (mon.0) 867 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.657984+0000 mon.a (mon.0) 868 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.668490+0000 mon.a (mon.0) 869 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.760244+0000 mon.a (mon.0) 870 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:32:15.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.763218+0000 mon.a (mon.0) 871 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:32:15.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.764911+0000 mon.a (mon.0) 872 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:32:15.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:14 smithi089 bash[31522]: audit 2024-04-13T07:32:14.774856+0000 mon.a (mon.0) 873 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:32:16.171 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:15 smithi089 bash[21353]: cluster 2024-04-13T07:32:14.891823+0000 mgr.y (mgr.14373) 118 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 417 B/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-13T07:32:16.171 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:15 smithi089 bash[31522]: cluster 2024-04-13T07:32:14.891823+0000 mgr.y (mgr.14373) 118 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 417 B/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-13T07:32:16.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:15 smithi203 bash[23480]: cluster 2024-04-13T07:32:14.891823+0000 mgr.y (mgr.14373) 118 : cluster 0 pgmap v91: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 417 B/s rd, 1.4 KiB/s wr, 4 op/s 2024-04-13T07:32:16.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:32:16 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:32:16] "GET /metrics HTTP/1.1" 200 33480 "" "Prometheus/2.43.0" 2024-04-13T07:32:17.059 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:18.147 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:17 smithi089 bash[31522]: cluster 2024-04-13T07:32:16.892891+0000 mgr.y (mgr.14373) 119 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-04-13T07:32:18.147 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:17 smithi089 bash[21353]: cluster 2024-04-13T07:32:16.892891+0000 mgr.y (mgr.14373) 119 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-04-13T07:32:18.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:17 smithi203 bash[23480]: cluster 2024-04-13T07:32:16.892891+0000 mgr.y (mgr.14373) 119 : cluster 0 pgmap v92: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-04-13T07:32:20.235 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:19 smithi089 bash[21353]: cluster 2024-04-13T07:32:18.893524+0000 mgr.y (mgr.14373) 120 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-04-13T07:32:20.235 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:19 smithi089 bash[31522]: cluster 2024-04-13T07:32:18.893524+0000 mgr.y (mgr.14373) 120 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-04-13T07:32:20.236 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-13T07:32:20.246 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-13T07:32:20.246 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-13T07:32:20.247 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph mgr dump --format=json 2024-04-13T07:32:20.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:19 smithi203 bash[23480]: cluster 2024-04-13T07:32:18.893524+0000 mgr.y (mgr.14373) 120 : cluster 0 pgmap v93: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 1.4 KiB/s wr, 12 op/s 2024-04-13T07:32:22.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:21 smithi203 bash[23480]: cluster 2024-04-13T07:32:20.894687+0000 mgr.y (mgr.14373) 121 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-13T07:32:22.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:21 smithi089 bash[31522]: cluster 2024-04-13T07:32:20.894687+0000 mgr.y (mgr.14373) 121 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-13T07:32:22.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:21 smithi089 bash[21353]: cluster 2024-04-13T07:32:20.894687+0000 mgr.y (mgr.14373) 121 : cluster 0 pgmap v94: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 8.0 KiB/s rd, 0 B/s wr, 7 op/s 2024-04-13T07:32:24.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:23 smithi203 bash[23480]: cluster 2024-04-13T07:32:22.895326+0000 mgr.y (mgr.14373) 122 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:24.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:23 smithi203 bash[23480]: audit 2024-04-13T07:32:23.932400+0000 mon.a (mon.0) 874 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:24.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:23 smithi089 bash[21353]: cluster 2024-04-13T07:32:22.895326+0000 mgr.y (mgr.14373) 122 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:24.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:23 smithi089 bash[21353]: audit 2024-04-13T07:32:23.932400+0000 mon.a (mon.0) 874 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:24.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:23 smithi089 bash[31522]: cluster 2024-04-13T07:32:22.895326+0000 mgr.y (mgr.14373) 122 : cluster 0 pgmap v95: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:24.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:23 smithi089 bash[31522]: audit 2024-04-13T07:32:23.932400+0000 mon.a (mon.0) 874 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:24.986 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:26.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:25 smithi089 bash[21353]: cluster 2024-04-13T07:32:24.896330+0000 mgr.y (mgr.14373) 123 : cluster 0 pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:26.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:25 smithi089 bash[31522]: cluster 2024-04-13T07:32:24.896330+0000 mgr.y (mgr.14373) 123 : cluster 0 pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:26.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:25 smithi203 bash[23480]: cluster 2024-04-13T07:32:24.896330+0000 mgr.y (mgr.14373) 123 : cluster 0 pgmap v96: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.7 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:26.527 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:32:26 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:32:26] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.43.0" 2024-04-13T07:32:27.627 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:32:28.178 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":20,"flags":0,"active_gid":14373,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":2897907290},{"type":"v1","addr":"172.21.15.89:6801","nonce":2897907290}]},"active_addr":"172.21.15.89:6801/2897907290","active_change":"2024-04-13T07:29:38.824930+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":24286,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.0.0","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3991952503}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1853920825}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":4124851388}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":4260385376}]}]} 2024-04-13T07:32:28.182 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-13T07:32:28.183 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-13T07:32:28.183 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-13T07:32:28.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:27 smithi203 bash[23480]: cluster 2024-04-13T07:32:26.897292+0000 mgr.y (mgr.14373) 124 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:28.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:27 smithi203 bash[23480]: audit 2024-04-13T07:32:27.624797+0000 mon.a (mon.0) 875 : audit 0 from='client.? 172.21.15.89:0/1366107614' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:32:28.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:27 smithi089 bash[21353]: cluster 2024-04-13T07:32:26.897292+0000 mgr.y (mgr.14373) 124 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:28.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:27 smithi089 bash[21353]: audit 2024-04-13T07:32:27.624797+0000 mon.a (mon.0) 875 : audit 0 from='client.? 172.21.15.89:0/1366107614' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:32:28.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:27 smithi089 bash[31522]: cluster 2024-04-13T07:32:26.897292+0000 mgr.y (mgr.14373) 124 : cluster 0 pgmap v97: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 6.3 KiB/s rd, 0 B/s wr, 6 op/s 2024-04-13T07:32:28.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:27 smithi089 bash[31522]: audit 2024-04-13T07:32:27.624797+0000 mon.a (mon.0) 875 : audit 0 from='client.? 172.21.15.89:0/1366107614' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:32:30.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:29 smithi203 bash[23480]: cluster 2024-04-13T07:32:28.897916+0000 mgr.y (mgr.14373) 125 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-13T07:32:30.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:29 smithi089 bash[21353]: cluster 2024-04-13T07:32:28.897916+0000 mgr.y (mgr.14373) 125 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-13T07:32:30.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:29 smithi089 bash[31522]: cluster 2024-04-13T07:32:28.897916+0000 mgr.y (mgr.14373) 125 : cluster 0 pgmap v98: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-13T07:32:32.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:31 smithi203 bash[23480]: cluster 2024-04-13T07:32:30.898608+0000 mgr.y (mgr.14373) 126 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-13T07:32:32.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:31 smithi089 bash[31522]: cluster 2024-04-13T07:32:30.898608+0000 mgr.y (mgr.14373) 126 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-13T07:32:32.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:31 smithi089 bash[21353]: cluster 2024-04-13T07:32:30.898608+0000 mgr.y (mgr.14373) 126 : cluster 0 pgmap v99: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2024-04-13T07:32:32.923 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:34.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:33 smithi203 bash[23480]: cluster 2024-04-13T07:32:32.899328+0000 mgr.y (mgr.14373) 127 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:34.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:33 smithi089 bash[21353]: cluster 2024-04-13T07:32:32.899328+0000 mgr.y (mgr.14373) 127 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:34.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:33 smithi089 bash[31522]: cluster 2024-04-13T07:32:32.899328+0000 mgr.y (mgr.14373) 127 : cluster 0 pgmap v100: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:35.376 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:32:35.377 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":70,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","created":"2024-04-13T07:19:16.936150+0000","modified":"2024-04-13T07:32:10.835206+0000","last_up_change":"2024-04-13T07:28:40.551327+0000","last_in_change":"2024-04-13T07:28:13.615937+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-13T07:24:24.430140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-13T07:30:20.226256+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-13T07:32:01.556724+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-13T07:32:02.440617+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9de691f5-25ed-472d-b6dd-dc3908fa85c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6803","nonce":3872923960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6805","nonce":3872923960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6809","nonce":3872923960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6807","nonce":3872923960}]},"public_addr":"172.21.15.89:6803/3872923960","cluster_addr":"172.21.15.89:6805/3872923960","heartbeat_back_addr":"172.21.15.89:6809/3872923960","heartbeat_front_addr":"172.21.15.89:6807/3872923960","state":["exists","up"]},{"osd":1,"uuid":"ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6811","nonce":302139657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6813","nonce":302139657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6817","nonce":302139657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6815","nonce":302139657}]},"public_addr":"172.21.15.89:6811/302139657","cluster_addr":"172.21.15.89:6813/302139657","heartbeat_back_addr":"172.21.15.89:6817/302139657","heartbeat_front_addr":"172.21.15.89:6815/302139657","state":["exists","up"]},{"osd":2,"uuid":"f86d6aea-97d3-4d75-99d0-6e5781d7a990","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6819","nonce":46732397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6821","nonce":46732397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6825","nonce":46732397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6823","nonce":46732397}]},"public_addr":"172.21.15.89:6819/46732397","cluster_addr":"172.21.15.89:6821/46732397","heartbeat_back_addr":"172.21.15.89:6825/46732397","heartbeat_front_addr":"172.21.15.89:6823/46732397","state":["exists","up"]},{"osd":3,"uuid":"37c263d3-d642-4cfa-99da-5a33e1bef16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6827","nonce":3306593060}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6829","nonce":3306593060}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6833","nonce":3306593060}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6831","nonce":3306593060}]},"public_addr":"172.21.15.89:6827/3306593060","cluster_addr":"172.21.15.89:6829/3306593060","heartbeat_back_addr":"172.21.15.89:6833/3306593060","heartbeat_front_addr":"172.21.15.89:6831/3306593060","state":["exists","up"]},{"osd":4,"uuid":"c1befaa2-c9f5-4023-9909-0577509ce44d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6801","nonce":3982510878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6803","nonce":3982510878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6807","nonce":3982510878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6805","nonce":3982510878}]},"public_addr":"172.21.15.203:6801/3982510878","cluster_addr":"172.21.15.203:6803/3982510878","heartbeat_back_addr":"172.21.15.203:6807/3982510878","heartbeat_front_addr":"172.21.15.203:6805/3982510878","state":["exists","up"]},{"osd":5,"uuid":"b45987d4-fc0f-45ad-9f0d-49240d898bfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6809","nonce":209623809}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6811","nonce":209623809}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6815","nonce":209623809}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6813","nonce":209623809}]},"public_addr":"172.21.15.203:6809/209623809","cluster_addr":"172.21.15.203:6811/209623809","heartbeat_back_addr":"172.21.15.203:6815/209623809","heartbeat_front_addr":"172.21.15.203:6813/209623809","state":["exists","up"]},{"osd":6,"uuid":"91bbf36b-799c-49df-8fbf-42a958699642","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6817","nonce":2138665907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6819","nonce":2138665907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6823","nonce":2138665907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6821","nonce":2138665907}]},"public_addr":"172.21.15.203:6817/2138665907","cluster_addr":"172.21.15.203:6819/2138665907","heartbeat_back_addr":"172.21.15.203:6823/2138665907","heartbeat_front_addr":"172.21.15.203:6821/2138665907","state":["exists","up"]},{"osd":7,"uuid":"aa9fb5c4-06e4-45ca-85f8-f2a96be15086","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6825","nonce":3848298771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6827","nonce":3848298771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6831","nonce":3848298771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6829","nonce":3848298771}]},"public_addr":"172.21.15.203:6825/3848298771","cluster_addr":"172.21.15.203:6827/3848298771","heartbeat_back_addr":"172.21.15.203:6831/3848298771","heartbeat_front_addr":"172.21.15.203:6829/3848298771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:22:41.252314+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:23:31.382195+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:24:20.791478+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:10.117632+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:59.130696+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:26:51.940707+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:27:44.966139+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:28:38.869265+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/1215893273":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/2483652022":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6801/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/868096906":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/3574204772":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6835/611307576":"2024-04-14T07:32:10.835116+0000","172.21.15.89:0/808783885":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2936084513":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/28202064":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6801/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/2680580613":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/1139384373":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6834/611307576":"2024-04-14T07:32:10.835116+0000","172.21.15.89:6801/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/3115322972":"2024-04-14T07:29:38.824647+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-13T07:32:35.984 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-13T07:32:35.984 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-13T07:32:36.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:35 smithi203 bash[23480]: cluster 2024-04-13T07:32:34.900238+0000 mgr.y (mgr.14373) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:36.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:35 smithi203 bash[23480]: audit 2024-04-13T07:32:35.382021+0000 mon.a (mon.0) 876 : audit 0 from='client.? 172.21.15.89:0/137037044' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:32:36.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:32:36 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:32:36] "GET /metrics HTTP/1.1" 200 33776 "" "Prometheus/2.43.0" 2024-04-13T07:32:36.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:35 smithi089 bash[21353]: cluster 2024-04-13T07:32:34.900238+0000 mgr.y (mgr.14373) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:36.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:35 smithi089 bash[21353]: audit 2024-04-13T07:32:35.382021+0000 mon.a (mon.0) 876 : audit 0 from='client.? 172.21.15.89:0/137037044' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:32:36.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:35 smithi089 bash[31522]: cluster 2024-04-13T07:32:34.900238+0000 mgr.y (mgr.14373) 128 : cluster 0 pgmap v101: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:36.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:35 smithi089 bash[31522]: audit 2024-04-13T07:32:35.382021+0000 mon.a (mon.0) 876 : audit 0 from='client.? 172.21.15.89:0/137037044' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:32:38.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:37 smithi203 bash[23480]: cluster 2024-04-13T07:32:36.901288+0000 mgr.y (mgr.14373) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:38.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:37 smithi089 bash[31522]: cluster 2024-04-13T07:32:36.901288+0000 mgr.y (mgr.14373) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:38.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:37 smithi089 bash[21353]: cluster 2024-04-13T07:32:36.901288+0000 mgr.y (mgr.14373) 129 : cluster 0 pgmap v102: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:39.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:38 smithi203 bash[23480]: audit 2024-04-13T07:32:38.887441+0000 mon.a (mon.0) 877 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-13T07:32:39.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:38 smithi203 bash[23480]: audit 2024-04-13T07:32:38.888180+0000 mon.a (mon.0) 878 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-13T07:32:39.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:38 smithi203 bash[23480]: audit 2024-04-13T07:32:38.888828+0000 mon.a (mon.0) 879 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-13T07:32:39.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:38 smithi203 bash[23480]: audit 2024-04-13T07:32:38.933018+0000 mon.a (mon.0) 880 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:39.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:38 smithi089 bash[21353]: audit 2024-04-13T07:32:38.887441+0000 mon.a (mon.0) 877 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-13T07:32:39.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:38 smithi089 bash[21353]: audit 2024-04-13T07:32:38.888180+0000 mon.a (mon.0) 878 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-13T07:32:39.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:38 smithi089 bash[21353]: audit 2024-04-13T07:32:38.888828+0000 mon.a (mon.0) 879 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-13T07:32:39.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:38 smithi089 bash[21353]: audit 2024-04-13T07:32:38.933018+0000 mon.a (mon.0) 880 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:39.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:38 smithi089 bash[31522]: audit 2024-04-13T07:32:38.887441+0000 mon.a (mon.0) 877 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2024-04-13T07:32:39.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:38 smithi089 bash[31522]: audit 2024-04-13T07:32:38.888180+0000 mon.a (mon.0) 878 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]: dispatch 2024-04-13T07:32:39.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:38 smithi089 bash[31522]: audit 2024-04-13T07:32:38.888828+0000 mon.a (mon.0) 879 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]: dispatch 2024-04-13T07:32:39.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:38 smithi089 bash[31522]: audit 2024-04-13T07:32:38.933018+0000 mon.a (mon.0) 880 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:40.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[21353]: cluster 2024-04-13T07:32:38.901943+0000 mgr.y (mgr.14373) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:40.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[21353]: audit 2024-04-13T07:32:39.013593+0000 mon.a (mon.0) 881 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-04-13T07:32:40.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[21353]: audit 2024-04-13T07:32:39.014000+0000 mon.a (mon.0) 882 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-13T07:32:40.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[21353]: audit 2024-04-13T07:32:39.014206+0000 mon.a (mon.0) 883 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-13T07:32:40.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[21353]: cluster 2024-04-13T07:32:39.014359+0000 mon.a (mon.0) 884 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-04-13T07:32:40.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[31522]: cluster 2024-04-13T07:32:38.901943+0000 mgr.y (mgr.14373) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:40.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[31522]: audit 2024-04-13T07:32:39.013593+0000 mon.a (mon.0) 881 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-04-13T07:32:40.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[31522]: audit 2024-04-13T07:32:39.014000+0000 mon.a (mon.0) 882 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-13T07:32:40.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[31522]: audit 2024-04-13T07:32:39.014206+0000 mon.a (mon.0) 883 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-13T07:32:40.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:40 smithi089 bash[31522]: cluster 2024-04-13T07:32:39.014359+0000 mon.a (mon.0) 884 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-04-13T07:32:40.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:40 smithi203 bash[23480]: cluster 2024-04-13T07:32:38.901943+0000 mgr.y (mgr.14373) 130 : cluster 0 pgmap v103: 73 pgs: 73 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:40 smithi203 bash[23480]: audit 2024-04-13T07:32:39.013593+0000 mon.a (mon.0) 881 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2024-04-13T07:32:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:40 smithi203 bash[23480]: audit 2024-04-13T07:32:39.014000+0000 mon.a (mon.0) 882 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [1, 5]}]': finished 2024-04-13T07:32:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:40 smithi203 bash[23480]: audit 2024-04-13T07:32:39.014206+0000 mon.a (mon.0) 883 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1d", "id": [1, 2]}]': finished 2024-04-13T07:32:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:40 smithi203 bash[23480]: cluster 2024-04-13T07:32:39.014359+0000 mon.a (mon.0) 884 : cluster 0 osdmap e71: 8 total, 8 up, 8 in 2024-04-13T07:32:40.724 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:41.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:41 smithi089 bash[31522]: cluster 2024-04-13T07:32:40.015840+0000 mon.a (mon.0) 885 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-04-13T07:32:41.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:41 smithi089 bash[21353]: cluster 2024-04-13T07:32:40.015840+0000 mon.a (mon.0) 885 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-04-13T07:32:41.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:41 smithi203 bash[23480]: cluster 2024-04-13T07:32:40.015840+0000 mon.a (mon.0) 885 : cluster 0 osdmap e72: 8 total, 8 up, 8 in 2024-04-13T07:32:42.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:42 smithi089 bash[31522]: cluster 2024-04-13T07:32:40.902562+0000 mgr.y (mgr.14373) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:42.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:42 smithi089 bash[21353]: cluster 2024-04-13T07:32:40.902562+0000 mgr.y (mgr.14373) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:42.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:42 smithi203 bash[23480]: cluster 2024-04-13T07:32:40.902562+0000 mgr.y (mgr.14373) 131 : cluster 0 pgmap v106: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:43.529 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:32:43.529 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":72,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","created":"2024-04-13T07:19:16.936150+0000","modified":"2024-04-13T07:32:40.005649+0000","last_up_change":"2024-04-13T07:28:40.551327+0000","last_in_change":"2024-04-13T07:28:13.615937+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-13T07:24:24.430140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-13T07:30:20.226256+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"64","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-13T07:32:01.556724+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-13T07:32:02.440617+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9de691f5-25ed-472d-b6dd-dc3908fa85c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6803","nonce":3872923960}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6805","nonce":3872923960}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6809","nonce":3872923960}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":3872923960},{"type":"v1","addr":"172.21.15.89:6807","nonce":3872923960}]},"public_addr":"172.21.15.89:6803/3872923960","cluster_addr":"172.21.15.89:6805/3872923960","heartbeat_back_addr":"172.21.15.89:6809/3872923960","heartbeat_front_addr":"172.21.15.89:6807/3872923960","state":["exists","up"]},{"osd":1,"uuid":"ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6811","nonce":302139657}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6813","nonce":302139657}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6817","nonce":302139657}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":302139657},{"type":"v1","addr":"172.21.15.89:6815","nonce":302139657}]},"public_addr":"172.21.15.89:6811/302139657","cluster_addr":"172.21.15.89:6813/302139657","heartbeat_back_addr":"172.21.15.89:6817/302139657","heartbeat_front_addr":"172.21.15.89:6815/302139657","state":["exists","up"]},{"osd":2,"uuid":"f86d6aea-97d3-4d75-99d0-6e5781d7a990","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6819","nonce":46732397}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6821","nonce":46732397}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6825","nonce":46732397}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":46732397},{"type":"v1","addr":"172.21.15.89:6823","nonce":46732397}]},"public_addr":"172.21.15.89:6819/46732397","cluster_addr":"172.21.15.89:6821/46732397","heartbeat_back_addr":"172.21.15.89:6825/46732397","heartbeat_front_addr":"172.21.15.89:6823/46732397","state":["exists","up"]},{"osd":3,"uuid":"37c263d3-d642-4cfa-99da-5a33e1bef16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":26,"up_thru":71,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6827","nonce":3306593060}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6829","nonce":3306593060}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6833","nonce":3306593060}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":3306593060},{"type":"v1","addr":"172.21.15.89:6831","nonce":3306593060}]},"public_addr":"172.21.15.89:6827/3306593060","cluster_addr":"172.21.15.89:6829/3306593060","heartbeat_back_addr":"172.21.15.89:6833/3306593060","heartbeat_front_addr":"172.21.15.89:6831/3306593060","state":["exists","up"]},{"osd":4,"uuid":"c1befaa2-c9f5-4023-9909-0577509ce44d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":31,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6801","nonce":3982510878}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6803","nonce":3982510878}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6807","nonce":3982510878}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":3982510878},{"type":"v1","addr":"172.21.15.203:6805","nonce":3982510878}]},"public_addr":"172.21.15.203:6801/3982510878","cluster_addr":"172.21.15.203:6803/3982510878","heartbeat_back_addr":"172.21.15.203:6807/3982510878","heartbeat_front_addr":"172.21.15.203:6805/3982510878","state":["exists","up"]},{"osd":5,"uuid":"b45987d4-fc0f-45ad-9f0d-49240d898bfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6809","nonce":209623809}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6811","nonce":209623809}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6815","nonce":209623809}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6813","nonce":209623809}]},"public_addr":"172.21.15.203:6809/209623809","cluster_addr":"172.21.15.203:6811/209623809","heartbeat_back_addr":"172.21.15.203:6815/209623809","heartbeat_front_addr":"172.21.15.203:6813/209623809","state":["exists","up"]},{"osd":6,"uuid":"91bbf36b-799c-49df-8fbf-42a958699642","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6817","nonce":2138665907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6819","nonce":2138665907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6823","nonce":2138665907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6821","nonce":2138665907}]},"public_addr":"172.21.15.203:6817/2138665907","cluster_addr":"172.21.15.203:6819/2138665907","heartbeat_back_addr":"172.21.15.203:6823/2138665907","heartbeat_front_addr":"172.21.15.203:6821/2138665907","state":["exists","up"]},{"osd":7,"uuid":"aa9fb5c4-06e4-45ca-85f8-f2a96be15086","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":65,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6825","nonce":3848298771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6827","nonce":3848298771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6831","nonce":3848298771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6829","nonce":3848298771}]},"public_addr":"172.21.15.203:6825/3848298771","cluster_addr":"172.21.15.203:6827/3848298771","heartbeat_back_addr":"172.21.15.203:6831/3848298771","heartbeat_front_addr":"172.21.15.203:6829/3848298771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:22:41.252314+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:23:31.382195+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:24:20.791478+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:10.117632+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:59.130696+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:26:51.940707+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:27:44.966139+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:28:38.869265+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/1215893273":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/2483652022":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6801/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/868096906":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/3574204772":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6835/611307576":"2024-04-14T07:32:10.835116+0000","172.21.15.89:0/808783885":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2936084513":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/28202064":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6801/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/2680580613":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/1139384373":"2024-04-14T07:29:38.824647+0000","172.21.15.89:6834/611307576":"2024-04-14T07:32:10.835116+0000","172.21.15.89:6801/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/3115322972":"2024-04-14T07:29:38.824647+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-13T07:32:44.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-13T07:32:44.141 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-13T07:32:44.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-13T07:32:44.142 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-13T07:32:44.143 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-13T07:32:44.143 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-13T07:32:44.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-13T07:32:44.144 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-13T07:32:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:44 smithi089 bash[31522]: cluster 2024-04-13T07:32:42.903190+0000 mgr.y (mgr.14373) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:44 smithi089 bash[31522]: audit 2024-04-13T07:32:43.534583+0000 mon.a (mon.0) 886 : audit 0 from='client.? 172.21.15.89:0/590647597' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:32:44.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:44 smithi089 bash[21353]: cluster 2024-04-13T07:32:42.903190+0000 mgr.y (mgr.14373) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:44 smithi089 bash[21353]: audit 2024-04-13T07:32:43.534583+0000 mon.a (mon.0) 886 : audit 0 from='client.? 172.21.15.89:0/590647597' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:32:44.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:44 smithi203 bash[23480]: cluster 2024-04-13T07:32:42.903190+0000 mgr.y (mgr.14373) 132 : cluster 0 pgmap v107: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 230 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:44.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:44 smithi203 bash[23480]: audit 2024-04-13T07:32:43.534583+0000 mon.a (mon.0) 886 : audit 0 from='client.? 172.21.15.89:0/590647597' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:32:46.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:46 smithi089 bash[21353]: cluster 2024-04-13T07:32:44.904189+0000 mgr.y (mgr.14373) 133 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:46.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:32:46 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:32:46] "GET /metrics HTTP/1.1" 200 33775 "" "Prometheus/2.43.0" 2024-04-13T07:32:46.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:46 smithi089 bash[31522]: cluster 2024-04-13T07:32:44.904189+0000 mgr.y (mgr.14373) 133 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:46.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:46 smithi203 bash[23480]: cluster 2024-04-13T07:32:44.904189+0000 mgr.y (mgr.14373) 133 : cluster 0 pgmap v108: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:48.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:48 smithi089 bash[31522]: cluster 2024-04-13T07:32:46.904810+0000 mgr.y (mgr.14373) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:48.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:48 smithi089 bash[21353]: cluster 2024-04-13T07:32:46.904810+0000 mgr.y (mgr.14373) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:48.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:48 smithi203 bash[23480]: cluster 2024-04-13T07:32:46.904810+0000 mgr.y (mgr.14373) 134 : cluster 0 pgmap v109: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:48.980 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:48.981 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:48.981 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:48.982 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:48.982 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:48.983 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:48.983 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:48.985 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:32:50.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:50 smithi089 bash[21353]: cluster 2024-04-13T07:32:48.905436+0000 mgr.y (mgr.14373) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:50.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:50 smithi089 bash[31522]: cluster 2024-04-13T07:32:48.905436+0000 mgr.y (mgr.14373) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:50.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:50 smithi203 bash[23480]: cluster 2024-04-13T07:32:48.905436+0000 mgr.y (mgr.14373) 135 : cluster 0 pgmap v110: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:52.331 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:52 smithi089 bash[31522]: cluster 2024-04-13T07:32:50.906970+0000 mgr.y (mgr.14373) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:52.331 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:52 smithi089 bash[21353]: cluster 2024-04-13T07:32:50.906970+0000 mgr.y (mgr.14373) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:52.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:52 smithi203 bash[23480]: cluster 2024-04-13T07:32:50.906970+0000 mgr.y (mgr.14373) 136 : cluster 0 pgmap v111: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:54.125 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:54 smithi089 bash[31522]: cluster 2024-04-13T07:32:52.907647+0000 mgr.y (mgr.14373) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:54.126 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:54 smithi089 bash[31522]: audit 2024-04-13T07:32:53.932536+0000 mon.a (mon.0) 887 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:54.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:54 smithi089 bash[21353]: cluster 2024-04-13T07:32:52.907647+0000 mgr.y (mgr.14373) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:54.126 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:54 smithi089 bash[21353]: audit 2024-04-13T07:32:53.932536+0000 mon.a (mon.0) 887 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:54.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:54 smithi203 bash[23480]: cluster 2024-04-13T07:32:52.907647+0000 mgr.y (mgr.14373) 137 : cluster 0 pgmap v112: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:54.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:54 smithi203 bash[23480]: audit 2024-04-13T07:32:53.932536+0000 mon.a (mon.0) 887 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:32:56.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:56 smithi089 bash[21353]: cluster 2024-04-13T07:32:54.908787+0000 mgr.y (mgr.14373) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:56.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:32:56 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:32:56] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.43.0" 2024-04-13T07:32:56.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:56 smithi089 bash[31522]: cluster 2024-04-13T07:32:54.908787+0000 mgr.y (mgr.14373) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:56.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:56 smithi203 bash[23480]: cluster 2024-04-13T07:32:54.908787+0000 mgr.y (mgr.14373) 138 : cluster 0 pgmap v113: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:58.351 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:32:58 smithi089 bash[21353]: cluster 2024-04-13T07:32:56.909392+0000 mgr.y (mgr.14373) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:58.352 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:32:58 smithi089 bash[31522]: cluster 2024-04-13T07:32:56.909392+0000 mgr.y (mgr.14373) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:58.353 INFO:teuthology.orchestra.run.smithi089.stdout:34359738494 2024-04-13T07:32:58.353 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-13T07:32:58.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:32:58 smithi203 bash[23480]: cluster 2024-04-13T07:32:56.909392+0000 mgr.y (mgr.14373) 139 : cluster 0 pgmap v114: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:32:59.846 INFO:teuthology.orchestra.run.smithi089.stdout:55834574963 2024-04-13T07:32:59.846 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-13T07:32:59.848 INFO:teuthology.orchestra.run.smithi089.stdout:201863462976 2024-04-13T07:32:59.848 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-13T07:33:00.039 INFO:teuthology.orchestra.run.smithi089.stdout:111669149791 2024-04-13T07:33:00.039 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-13T07:33:00.220 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:00 smithi089 bash[21353]: cluster 2024-04-13T07:32:58.910321+0000 mgr.y (mgr.14373) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:00.220 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:00 smithi089 bash[31522]: cluster 2024-04-13T07:32:58.910321+0000 mgr.y (mgr.14373) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:00.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:00 smithi203 bash[23480]: cluster 2024-04-13T07:32:58.910321+0000 mgr.y (mgr.14373) 140 : cluster 0 pgmap v115: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:01.623 INFO:teuthology.orchestra.run.smithi089.stdout:236223201335 2024-04-13T07:33:01.624 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-13T07:33:01.688 INFO:teuthology.orchestra.run.smithi089.stdout:133143986262 2024-04-13T07:33:01.688 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-13T07:33:01.757 INFO:teuthology.orchestra.run.smithi089.stdout:167503724620 2024-04-13T07:33:01.757 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-13T07:33:01.820 INFO:teuthology.orchestra.run.smithi089.stdout:77309411434 2024-04-13T07:33:01.820 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-13T07:33:02.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:02 smithi203 bash[23480]: cluster 2024-04-13T07:33:00.911512+0000 mgr.y (mgr.14373) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:02.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:02 smithi089 bash[31522]: cluster 2024-04-13T07:33:00.911512+0000 mgr.y (mgr.14373) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:02.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:02 smithi089 bash[21353]: cluster 2024-04-13T07:33:00.911512+0000 mgr.y (mgr.14373) 141 : cluster 0 pgmap v116: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:04.344 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:04.378 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:04 smithi089 bash[21353]: cluster 2024-04-13T07:33:02.912107+0000 mgr.y (mgr.14373) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:04.378 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:04 smithi089 bash[31522]: cluster 2024-04-13T07:33:02.912107+0000 mgr.y (mgr.14373) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:04.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:04 smithi203 bash[23480]: cluster 2024-04-13T07:33:02.912107+0000 mgr.y (mgr.14373) 142 : cluster 0 pgmap v117: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:05.359 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:05.359 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:05.360 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:06.386 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:06 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:33:06] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.43.0" 2024-04-13T07:33:06.386 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:06 smithi089 bash[21353]: cluster 2024-04-13T07:33:04.913093+0000 mgr.y (mgr.14373) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:06.386 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:06 smithi089 bash[31522]: cluster 2024-04-13T07:33:04.913093+0000 mgr.y (mgr.14373) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:06.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:06 smithi203 bash[23480]: cluster 2024-04-13T07:33:04.913093+0000 mgr.y (mgr.14373) 143 : cluster 0 pgmap v118: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:06.681 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:06.682 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:06.682 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:06.682 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:08.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:08 smithi089 bash[31522]: cluster 2024-04-13T07:33:06.913671+0000 mgr.y (mgr.14373) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:08.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:08 smithi089 bash[21353]: cluster 2024-04-13T07:33:06.913671+0000 mgr.y (mgr.14373) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:08.319 INFO:teuthology.orchestra.run.smithi089.stdout:34359738496 2024-04-13T07:33:08.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:08 smithi203 bash[23480]: cluster 2024-04-13T07:33:06.913671+0000 mgr.y (mgr.14373) 144 : cluster 0 pgmap v119: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:09.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:09 smithi089 bash[31522]: audit 2024-04-13T07:33:08.327648+0000 mon.a (mon.0) 888 : audit 0 from='client.? 172.21.15.89:0/677075269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:33:09.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:09 smithi089 bash[31522]: audit 2024-04-13T07:33:08.933916+0000 mon.a (mon.0) 889 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:09.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:09 smithi089 bash[21353]: audit 2024-04-13T07:33:08.327648+0000 mon.a (mon.0) 888 : audit 0 from='client.? 172.21.15.89:0/677075269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:33:09.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:09 smithi089 bash[21353]: audit 2024-04-13T07:33:08.933916+0000 mon.a (mon.0) 889 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:09.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:09 smithi203 bash[23480]: audit 2024-04-13T07:33:08.327648+0000 mon.a (mon.0) 888 : audit 0 from='client.? 172.21.15.89:0/677075269' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:33:09.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:09 smithi203 bash[23480]: audit 2024-04-13T07:33:08.933916+0000 mon.a (mon.0) 889 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:10.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:10 smithi089 bash[31522]: cluster 2024-04-13T07:33:08.914371+0000 mgr.y (mgr.14373) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:10.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:10 smithi089 bash[21353]: cluster 2024-04-13T07:33:08.914371+0000 mgr.y (mgr.14373) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:10.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:10 smithi203 bash[23480]: cluster 2024-04-13T07:33:08.914371+0000 mgr.y (mgr.14373) 145 : cluster 0 pgmap v120: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:11.592 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738494 got 34359738496 for osd.0 2024-04-13T07:33:11.592 DEBUG:teuthology.parallel:result is None 2024-04-13T07:33:12.106 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:12 smithi089 bash[21353]: cluster 2024-04-13T07:33:10.915524+0000 mgr.y (mgr.14373) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:12.402 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:12 smithi089 bash[31522]: cluster 2024-04-13T07:33:10.915524+0000 mgr.y (mgr.14373) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:12.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:12 smithi203 bash[23480]: cluster 2024-04-13T07:33:10.915524+0000 mgr.y (mgr.14373) 146 : cluster 0 pgmap v121: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:14.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:14 smithi203 bash[23480]: cluster 2024-04-13T07:33:12.915940+0000 mgr.y (mgr.14373) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:14.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:14 smithi089 bash[21353]: cluster 2024-04-13T07:33:12.915940+0000 mgr.y (mgr.14373) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:14.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:14 smithi089 bash[31522]: cluster 2024-04-13T07:33:12.915940+0000 mgr.y (mgr.14373) 147 : cluster 0 pgmap v122: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:15.159 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:15 smithi089 bash[21353]: audit 2024-04-13T07:33:14.865719+0000 mon.a (mon.0) 890 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:33:15.159 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:15 smithi089 bash[31522]: audit 2024-04-13T07:33:14.865719+0000 mon.a (mon.0) 890 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:33:15.311 INFO:teuthology.orchestra.run.smithi089.stdout:111669149795 2024-04-13T07:33:15.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:15 smithi203 bash[23480]: audit 2024-04-13T07:33:14.865719+0000 mon.a (mon.0) 890 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:33:15.520 INFO:teuthology.orchestra.run.smithi089.stdout:201863462980 2024-04-13T07:33:15.886 INFO:teuthology.orchestra.run.smithi089.stdout:167503724623 2024-04-13T07:33:15.898 INFO:teuthology.orchestra.run.smithi089.stdout:55834574967 2024-04-13T07:33:16.262 INFO:teuthology.orchestra.run.smithi089.stdout:77309411437 2024-04-13T07:33:16.396 INFO:teuthology.orchestra.run.smithi089.stdout:133143986265 2024-04-13T07:33:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:16 smithi203 bash[23480]: cluster 2024-04-13T07:33:14.916524+0000 mgr.y (mgr.14373) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:16 smithi203 bash[23480]: audit 2024-04-13T07:33:15.318919+0000 mon.a (mon.0) 891 : audit 0 from='client.? 172.21.15.89:0/611926763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:33:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:16 smithi203 bash[23480]: audit 2024-04-13T07:33:15.401492+0000 mon.a (mon.0) 892 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:33:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:16 smithi203 bash[23480]: audit 2024-04-13T07:33:15.402215+0000 mon.a (mon.0) 893 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:33:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:16 smithi203 bash[23480]: audit 2024-04-13T07:33:15.407349+0000 mon.a (mon.0) 894 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:16 smithi203 bash[23480]: audit 2024-04-13T07:33:15.528352+0000 mon.a (mon.0) 895 : audit 0 from='client.? 172.21.15.89:0/2455054382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:33:16.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:16 smithi203 bash[23480]: audit 2024-04-13T07:33:15.893832+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.89:0/1992266445' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:33:16.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:16 smithi203 bash[23480]: audit 2024-04-13T07:33:15.905983+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.89:0/3145991682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:33:16.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:33:16] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.43.0" 2024-04-13T07:33:16.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[31522]: cluster 2024-04-13T07:33:14.916524+0000 mgr.y (mgr.14373) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:16.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[31522]: audit 2024-04-13T07:33:15.318919+0000 mon.a (mon.0) 891 : audit 0 from='client.? 172.21.15.89:0/611926763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[31522]: audit 2024-04-13T07:33:15.401492+0000 mon.a (mon.0) 892 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[31522]: audit 2024-04-13T07:33:15.402215+0000 mon.a (mon.0) 893 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[31522]: audit 2024-04-13T07:33:15.407349+0000 mon.a (mon.0) 894 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[31522]: audit 2024-04-13T07:33:15.528352+0000 mon.a (mon.0) 895 : audit 0 from='client.? 172.21.15.89:0/2455054382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[31522]: audit 2024-04-13T07:33:15.893832+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.89:0/1992266445' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[31522]: audit 2024-04-13T07:33:15.905983+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.89:0/3145991682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21353]: cluster 2024-04-13T07:33:14.916524+0000 mgr.y (mgr.14373) 148 : cluster 0 pgmap v123: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21353]: audit 2024-04-13T07:33:15.318919+0000 mon.a (mon.0) 891 : audit 0 from='client.? 172.21.15.89:0/611926763' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21353]: audit 2024-04-13T07:33:15.401492+0000 mon.a (mon.0) 892 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21353]: audit 2024-04-13T07:33:15.402215+0000 mon.a (mon.0) 893 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21353]: audit 2024-04-13T07:33:15.407349+0000 mon.a (mon.0) 894 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21353]: audit 2024-04-13T07:33:15.528352+0000 mon.a (mon.0) 895 : audit 0 from='client.? 172.21.15.89:0/2455054382' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:33:16.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21353]: audit 2024-04-13T07:33:15.893832+0000 mon.c (mon.2) 22 : audit 0 from='client.? 172.21.15.89:0/1992266445' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:33:16.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:16 smithi089 bash[21353]: audit 2024-04-13T07:33:15.905983+0000 mon.c (mon.2) 23 : audit 0 from='client.? 172.21.15.89:0/3145991682' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:33:17.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:17 smithi089 bash[21353]: audit 2024-04-13T07:33:16.269243+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.89:0/2814601556' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:33:17.394 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:17 smithi089 bash[21353]: audit 2024-04-13T07:33:16.403325+0000 mon.a (mon.0) 896 : audit 0 from='client.? 172.21.15.89:0/1330288802' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:33:17.394 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:17 smithi089 bash[31522]: audit 2024-04-13T07:33:16.269243+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.89:0/2814601556' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:33:17.394 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:17 smithi089 bash[31522]: audit 2024-04-13T07:33:16.403325+0000 mon.a (mon.0) 896 : audit 0 from='client.? 172.21.15.89:0/1330288802' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:33:17.395 INFO:tasks.cephadm.ceph_manager.ceph:need seq 111669149791 got 111669149795 for osd.3 2024-04-13T07:33:17.395 DEBUG:teuthology.parallel:result is None 2024-04-13T07:33:17.468 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863462976 got 201863462980 for osd.6 2024-04-13T07:33:17.468 DEBUG:teuthology.parallel:result is None 2024-04-13T07:33:17.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:17 smithi203 bash[23480]: audit 2024-04-13T07:33:16.269243+0000 mon.c (mon.2) 24 : audit 0 from='client.? 172.21.15.89:0/2814601556' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:33:17.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:17 smithi203 bash[23480]: audit 2024-04-13T07:33:16.403325+0000 mon.a (mon.0) 896 : audit 0 from='client.? 172.21.15.89:0/1330288802' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:33:17.546 INFO:teuthology.orchestra.run.smithi089.stdout:236223201338 2024-04-13T07:33:18.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:18 smithi203 bash[23480]: cluster 2024-04-13T07:33:16.917122+0000 mgr.y (mgr.14373) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:18.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:18 smithi203 bash[23480]: audit 2024-04-13T07:33:17.553975+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.89:0/1835339106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:33:18.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:18 smithi089 bash[21353]: cluster 2024-04-13T07:33:16.917122+0000 mgr.y (mgr.14373) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:18.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:18 smithi089 bash[21353]: audit 2024-04-13T07:33:17.553975+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.89:0/1835339106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:33:18.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:18 smithi089 bash[31522]: cluster 2024-04-13T07:33:16.917122+0000 mgr.y (mgr.14373) 149 : cluster 0 pgmap v124: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:18.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:18 smithi089 bash[31522]: audit 2024-04-13T07:33:17.553975+0000 mon.c (mon.2) 25 : audit 0 from='client.? 172.21.15.89:0/1835339106' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:33:18.693 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574963 got 55834574967 for osd.1 2024-04-13T07:33:18.693 DEBUG:teuthology.parallel:result is None 2024-04-13T07:33:18.799 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411434 got 77309411437 for osd.2 2024-04-13T07:33:18.799 DEBUG:teuthology.parallel:result is None 2024-04-13T07:33:18.942 INFO:tasks.cephadm.ceph_manager.ceph:need seq 133143986262 got 133143986265 for osd.4 2024-04-13T07:33:18.942 DEBUG:teuthology.parallel:result is None 2024-04-13T07:33:19.393 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724620 got 167503724623 for osd.5 2024-04-13T07:33:19.393 DEBUG:teuthology.parallel:result is None 2024-04-13T07:33:19.421 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201335 got 236223201338 for osd.7 2024-04-13T07:33:19.421 DEBUG:teuthology.parallel:result is None 2024-04-13T07:33:19.422 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-13T07:33:19.422 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-13T07:33:20.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:20 smithi203 bash[23480]: cluster 2024-04-13T07:33:18.917574+0000 mgr.y (mgr.14373) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:20.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:20 smithi089 bash[31522]: cluster 2024-04-13T07:33:18.917574+0000 mgr.y (mgr.14373) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:20.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:20 smithi089 bash[21353]: cluster 2024-04-13T07:33:18.917574+0000 mgr.y (mgr.14373) 150 : cluster 0 pgmap v125: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:22.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:22 smithi203 bash[23480]: cluster 2024-04-13T07:33:20.918740+0000 mgr.y (mgr.14373) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:22.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:22 smithi089 bash[21353]: cluster 2024-04-13T07:33:20.918740+0000 mgr.y (mgr.14373) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:22.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:22 smithi089 bash[31522]: cluster 2024-04-13T07:33:20.918740+0000 mgr.y (mgr.14373) 151 : cluster 0 pgmap v126: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:24.190 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:24.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:24 smithi203 bash[23480]: cluster 2024-04-13T07:33:22.919534+0000 mgr.y (mgr.14373) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:24.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:24 smithi203 bash[23480]: audit 2024-04-13T07:33:23.934047+0000 mon.a (mon.0) 897 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:24.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:24 smithi089 bash[21353]: cluster 2024-04-13T07:33:22.919534+0000 mgr.y (mgr.14373) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:24.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:24 smithi089 bash[21353]: audit 2024-04-13T07:33:23.934047+0000 mon.a (mon.0) 897 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:24.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:24 smithi089 bash[31522]: cluster 2024-04-13T07:33:22.919534+0000 mgr.y (mgr.14373) 152 : cluster 0 pgmap v127: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:24.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:24 smithi089 bash[31522]: audit 2024-04-13T07:33:23.934047+0000 mon.a (mon.0) 897 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:26.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:26 smithi203 bash[23480]: cluster 2024-04-13T07:33:24.920625+0000 mgr.y (mgr.14373) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:26.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:26 smithi089 bash[21353]: cluster 2024-04-13T07:33:24.920625+0000 mgr.y (mgr.14373) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:26.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:26 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:33:26] "GET /metrics HTTP/1.1" 200 33770 "" "Prometheus/2.43.0" 2024-04-13T07:33:26.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:26 smithi089 bash[31522]: cluster 2024-04-13T07:33:24.920625+0000 mgr.y (mgr.14373) 153 : cluster 0 pgmap v128: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:26.732 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:33:26.734 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-04-13T07:33:27.342 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":128,"stamp":"2024-04-13T07:33:24.919849+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592730,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":289,"num_read_kb":386,"num_write":224,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":236084,"kb_used_data":7616,"kb_used_omap":12,"kb_used_meta":228339,"kb_avail":749725132,"statfs":{"total":767960285184,"available":767718535168,"internally_reserved":0,"allocated":7798784,"data_stored":4373871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":233819705},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004110"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021925+0000","last_change":"2024-04-13T07:32:03.142678+0000","last_active":"2024-04-13T07:32:39.021925+0000","last_peered":"2024-04-13T07:32:39.021925+0000","last_clean":"2024-04-13T07:32:39.021925+0000","last_became_active":"2024-04-13T07:32:03.141871+0000","last_became_peered":"2024-04-13T07:32:03.141871+0000","last_unstale":"2024-04-13T07:32:39.021925+0000","last_undegraded":"2024-04-13T07:32:39.021925+0000","last_fullsized":"2024-04-13T07:32:39.021925+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:10:35.700451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013869+0000","last_change":"2024-04-13T07:32:03.086097+0000","last_active":"2024-04-13T07:32:39.013869+0000","last_peered":"2024-04-13T07:32:39.013869+0000","last_clean":"2024-04-13T07:32:39.013869+0000","last_became_active":"2024-04-13T07:32:03.066927+0000","last_became_peered":"2024-04-13T07:32:03.066927+0000","last_unstale":"2024-04-13T07:32:39.013869+0000","last_undegraded":"2024-04-13T07:32:39.013869+0000","last_fullsized":"2024-04-13T07:32:39.013869+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:39:35.420226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017118+0000","last_change":"2024-04-13T07:32:04.418359+0000","last_active":"2024-04-13T07:32:39.017118+0000","last_peered":"2024-04-13T07:32:39.017118+0000","last_clean":"2024-04-13T07:32:39.017118+0000","last_became_active":"2024-04-13T07:32:04.418051+0000","last_became_peered":"2024-04-13T07:32:04.418051+0000","last_unstale":"2024-04-13T07:32:39.017118+0000","last_undegraded":"2024-04-13T07:32:39.017118+0000","last_fullsized":"2024-04-13T07:32:39.017118+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:54:44.282519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.018012+0000","last_change":"2024-04-13T07:32:03.080811+0000","last_active":"2024-04-13T07:32:39.018012+0000","last_peered":"2024-04-13T07:32:39.018012+0000","last_clean":"2024-04-13T07:32:39.018012+0000","last_became_active":"2024-04-13T07:32:03.072598+0000","last_became_peered":"2024-04-13T07:32:03.072598+0000","last_unstale":"2024-04-13T07:32:39.018012+0000","last_undegraded":"2024-04-13T07:32:39.018012+0000","last_fullsized":"2024-04-13T07:32:39.018012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:13:04.105633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023060+0000","last_change":"2024-04-13T07:32:03.141148+0000","last_active":"2024-04-13T07:32:39.023060+0000","last_peered":"2024-04-13T07:32:39.023060+0000","last_clean":"2024-04-13T07:32:39.023060+0000","last_became_active":"2024-04-13T07:32:03.139990+0000","last_became_peered":"2024-04-13T07:32:03.139990+0000","last_unstale":"2024-04-13T07:32:39.023060+0000","last_undegraded":"2024-04-13T07:32:39.023060+0000","last_fullsized":"2024-04-13T07:32:39.023060+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:18:49.325969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022716+0000","last_change":"2024-04-13T07:32:03.077062+0000","last_active":"2024-04-13T07:32:39.022716+0000","last_peered":"2024-04-13T07:32:39.022716+0000","last_clean":"2024-04-13T07:32:39.022716+0000","last_became_active":"2024-04-13T07:32:03.076337+0000","last_became_peered":"2024-04-13T07:32:03.076337+0000","last_unstale":"2024-04-13T07:32:39.022716+0000","last_undegraded":"2024-04-13T07:32:39.022716+0000","last_fullsized":"2024-04-13T07:32:39.022716+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:22:01.092239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022844+0000","last_change":"2024-04-13T07:32:03.144389+0000","last_active":"2024-04-13T07:32:39.022844+0000","last_peered":"2024-04-13T07:32:39.022844+0000","last_clean":"2024-04-13T07:32:39.022844+0000","last_became_active":"2024-04-13T07:32:03.143541+0000","last_became_peered":"2024-04-13T07:32:03.143541+0000","last_unstale":"2024-04-13T07:32:39.022844+0000","last_undegraded":"2024-04-13T07:32:39.022844+0000","last_fullsized":"2024-04-13T07:32:39.022844+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:14:17.879743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021970+0000","last_change":"2024-04-13T07:32:03.066107+0000","last_active":"2024-04-13T07:32:39.021970+0000","last_peered":"2024-04-13T07:32:39.021970+0000","last_clean":"2024-04-13T07:32:39.021970+0000","last_became_active":"2024-04-13T07:32:03.065235+0000","last_became_peered":"2024-04-13T07:32:03.065235+0000","last_unstale":"2024-04-13T07:32:39.021970+0000","last_undegraded":"2024-04-13T07:32:39.021970+0000","last_fullsized":"2024-04-13T07:32:39.021970+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:46:26.415643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.024063+0000","last_change":"2024-04-13T07:32:03.141938+0000","last_active":"2024-04-13T07:32:39.024063+0000","last_peered":"2024-04-13T07:32:39.024063+0000","last_clean":"2024-04-13T07:32:39.024063+0000","last_became_active":"2024-04-13T07:32:03.141631+0000","last_became_peered":"2024-04-13T07:32:03.141631+0000","last_unstale":"2024-04-13T07:32:39.024063+0000","last_undegraded":"2024-04-13T07:32:39.024063+0000","last_fullsized":"2024-04-13T07:32:39.024063+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:42:20.084568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010665+0000","last_change":"2024-04-13T07:32:03.083518+0000","last_active":"2024-04-13T07:32:39.010665+0000","last_peered":"2024-04-13T07:32:39.010665+0000","last_clean":"2024-04-13T07:32:39.010665+0000","last_became_active":"2024-04-13T07:32:03.082695+0000","last_became_peered":"2024-04-13T07:32:03.082695+0000","last_unstale":"2024-04-13T07:32:39.010665+0000","last_undegraded":"2024-04-13T07:32:39.010665+0000","last_fullsized":"2024-04-13T07:32:39.010665+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:58:25.148756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022948+0000","last_change":"2024-04-13T07:32:03.140777+0000","last_active":"2024-04-13T07:32:39.022948+0000","last_peered":"2024-04-13T07:32:39.022948+0000","last_clean":"2024-04-13T07:32:39.022948+0000","last_became_active":"2024-04-13T07:32:03.140026+0000","last_became_peered":"2024-04-13T07:32:03.140026+0000","last_unstale":"2024-04-13T07:32:39.022948+0000","last_undegraded":"2024-04-13T07:32:39.022948+0000","last_fullsized":"2024-04-13T07:32:39.022948+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:03:27.099045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.011729+0000","last_change":"2024-04-13T07:32:03.063483+0000","last_active":"2024-04-13T07:32:39.011729+0000","last_peered":"2024-04-13T07:32:39.011729+0000","last_clean":"2024-04-13T07:32:39.011729+0000","last_became_active":"2024-04-13T07:32:03.062230+0000","last_became_peered":"2024-04-13T07:32:03.062230+0000","last_unstale":"2024-04-13T07:32:39.011729+0000","last_undegraded":"2024-04-13T07:32:39.011729+0000","last_fullsized":"2024-04-13T07:32:39.011729+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:40:08.222569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148756+0000","last_change":"2024-04-13T07:30:21.470196+0000","last_active":"2024-04-13T07:32:39.148756+0000","last_peered":"2024-04-13T07:32:39.148756+0000","last_clean":"2024-04-13T07:32:39.148756+0000","last_became_active":"2024-04-13T07:30:21.469340+0000","last_became_peered":"2024-04-13T07:30:21.469340+0000","last_unstale":"2024-04-13T07:32:39.148756+0000","last_undegraded":"2024-04-13T07:32:39.148756+0000","last_fullsized":"2024-04-13T07:32:39.148756+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:32:48.529436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.016477+0000","last_change":"2024-04-13T07:32:04.424228+0000","last_active":"2024-04-13T07:32:39.016477+0000","last_peered":"2024-04-13T07:32:39.016477+0000","last_clean":"2024-04-13T07:32:39.016477+0000","last_became_active":"2024-04-13T07:32:04.424027+0000","last_became_peered":"2024-04-13T07:32:04.424027+0000","last_unstale":"2024-04-13T07:32:39.016477+0000","last_undegraded":"2024-04-13T07:32:39.016477+0000","last_fullsized":"2024-04-13T07:32:39.016477+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:53:59.376447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.019359+0000","last_change":"2024-04-13T07:30:22.257370+0000","last_active":"2024-04-13T07:32:39.019359+0000","last_peered":"2024-04-13T07:32:39.019359+0000","last_clean":"2024-04-13T07:32:39.019359+0000","last_became_active":"2024-04-13T07:30:22.256904+0000","last_became_peered":"2024-04-13T07:30:22.256904+0000","last_unstale":"2024-04-13T07:32:39.019359+0000","last_undegraded":"2024-04-13T07:32:39.019359+0000","last_fullsized":"2024-04-13T07:32:39.019359+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:09:42.362257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.019750+0000","last_change":"2024-04-13T07:32:03.092985+0000","last_active":"2024-04-13T07:32:39.019750+0000","last_peered":"2024-04-13T07:32:39.019750+0000","last_clean":"2024-04-13T07:32:39.019750+0000","last_became_active":"2024-04-13T07:32:03.092332+0000","last_became_peered":"2024-04-13T07:32:03.092332+0000","last_unstale":"2024-04-13T07:32:39.019750+0000","last_undegraded":"2024-04-13T07:32:39.019750+0000","last_fullsized":"2024-04-13T07:32:39.019750+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:06:46.635980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010276+0000","last_change":"2024-04-13T07:32:04.425960+0000","last_active":"2024-04-13T07:32:39.010276+0000","last_peered":"2024-04-13T07:32:39.010276+0000","last_clean":"2024-04-13T07:32:39.010276+0000","last_became_active":"2024-04-13T07:32:04.425616+0000","last_became_peered":"2024-04-13T07:32:04.425616+0000","last_unstale":"2024-04-13T07:32:39.010276+0000","last_undegraded":"2024-04-13T07:32:39.010276+0000","last_fullsized":"2024-04-13T07:32:39.010276+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:10:37.088266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013622+0000","last_change":"2024-04-13T07:32:03.080050+0000","last_active":"2024-04-13T07:32:39.013622+0000","last_peered":"2024-04-13T07:32:39.013622+0000","last_clean":"2024-04-13T07:32:39.013622+0000","last_became_active":"2024-04-13T07:32:03.078878+0000","last_became_peered":"2024-04-13T07:32:03.078878+0000","last_unstale":"2024-04-13T07:32:39.013622+0000","last_undegraded":"2024-04-13T07:32:39.013622+0000","last_fullsized":"2024-04-13T07:32:39.013622+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:10:05.791661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010138+0000","last_change":"2024-04-13T07:30:22.258054+0000","last_active":"2024-04-13T07:32:39.010138+0000","last_peered":"2024-04-13T07:32:39.010138+0000","last_clean":"2024-04-13T07:32:39.010138+0000","last_became_active":"2024-04-13T07:30:22.256961+0000","last_became_peered":"2024-04-13T07:30:22.256961+0000","last_unstale":"2024-04-13T07:32:39.010138+0000","last_undegraded":"2024-04-13T07:32:39.010138+0000","last_fullsized":"2024-04-13T07:32:39.010138+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:13:36.989961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.012043+0000","last_change":"2024-04-13T07:32:04.418392+0000","last_active":"2024-04-13T07:32:39.012043+0000","last_peered":"2024-04-13T07:32:39.012043+0000","last_clean":"2024-04-13T07:32:39.012043+0000","last_became_active":"2024-04-13T07:32:04.418051+0000","last_became_peered":"2024-04-13T07:32:04.418051+0000","last_unstale":"2024-04-13T07:32:39.012043+0000","last_undegraded":"2024-04-13T07:32:39.012043+0000","last_fullsized":"2024-04-13T07:32:39.012043+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:43:31.028251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009957+0000","last_change":"2024-04-13T07:32:03.069788+0000","last_active":"2024-04-13T07:32:39.009957+0000","last_peered":"2024-04-13T07:32:39.009957+0000","last_clean":"2024-04-13T07:32:39.009957+0000","last_became_active":"2024-04-13T07:32:03.068886+0000","last_became_peered":"2024-04-13T07:32:03.068886+0000","last_unstale":"2024-04-13T07:32:39.009957+0000","last_undegraded":"2024-04-13T07:32:39.009957+0000","last_fullsized":"2024-04-13T07:32:39.009957+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:41:50.507675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009382+0000","last_change":"2024-04-13T07:30:21.472711+0000","last_active":"2024-04-13T07:32:39.009382+0000","last_peered":"2024-04-13T07:32:39.009382+0000","last_clean":"2024-04-13T07:32:39.009382+0000","last_became_active":"2024-04-13T07:30:21.471979+0000","last_became_peered":"2024-04-13T07:30:21.471979+0000","last_unstale":"2024-04-13T07:32:39.009382+0000","last_undegraded":"2024-04-13T07:32:39.009382+0000","last_fullsized":"2024-04-13T07:32:39.009382+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:27:07.929194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148712+0000","last_change":"2024-04-13T07:32:04.423737+0000","last_active":"2024-04-13T07:32:39.148712+0000","last_peered":"2024-04-13T07:32:39.148712+0000","last_clean":"2024-04-13T07:32:39.148712+0000","last_became_active":"2024-04-13T07:32:04.421863+0000","last_became_peered":"2024-04-13T07:32:04.421863+0000","last_unstale":"2024-04-13T07:32:39.148712+0000","last_undegraded":"2024-04-13T07:32:39.148712+0000","last_fullsized":"2024-04-13T07:32:39.148712+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:12:20.475957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017110+0000","last_change":"2024-04-13T07:32:03.072842+0000","last_active":"2024-04-13T07:32:39.017110+0000","last_peered":"2024-04-13T07:32:39.017110+0000","last_clean":"2024-04-13T07:32:39.017110+0000","last_became_active":"2024-04-13T07:32:03.071974+0000","last_became_peered":"2024-04-13T07:32:03.071974+0000","last_unstale":"2024-04-13T07:32:39.017110+0000","last_undegraded":"2024-04-13T07:32:39.017110+0000","last_fullsized":"2024-04-13T07:32:39.017110+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:56:10.031268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":269,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-04-13T07:32:40.238221+0000","last_change":"2024-04-13T07:28:43.834029+0000","last_active":"2024-04-13T07:32:40.238221+0000","last_peered":"2024-04-13T07:32:40.238221+0000","last_clean":"2024-04-13T07:32:40.238221+0000","last_became_active":"2024-04-13T07:28:43.833192+0000","last_became_peered":"2024-04-13T07:28:43.833192+0000","last_unstale":"2024-04-13T07:32:40.238221+0000","last_undegraded":"2024-04-13T07:32:40.238221+0000","last_fullsized":"2024-04-13T07:32:40.238221+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_clean_scrub_stamp":"2024-04-13T07:24:24.657602+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:32:17.256871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021717+0000","last_change":"2024-04-13T07:30:21.467433+0000","last_active":"2024-04-13T07:32:39.021717+0000","last_peered":"2024-04-13T07:32:39.021717+0000","last_clean":"2024-04-13T07:32:39.021717+0000","last_became_active":"2024-04-13T07:30:21.466602+0000","last_became_peered":"2024-04-13T07:30:21.466602+0000","last_unstale":"2024-04-13T07:32:39.021717+0000","last_undegraded":"2024-04-13T07:32:39.021717+0000","last_fullsized":"2024-04-13T07:32:39.021717+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:29:15.004301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.016883+0000","last_change":"2024-04-13T07:32:04.102888+0000","last_active":"2024-04-13T07:32:39.016883+0000","last_peered":"2024-04-13T07:32:39.016883+0000","last_clean":"2024-04-13T07:32:39.016883+0000","last_became_active":"2024-04-13T07:32:04.102086+0000","last_became_peered":"2024-04-13T07:32:04.102086+0000","last_unstale":"2024-04-13T07:32:39.016883+0000","last_undegraded":"2024-04-13T07:32:39.016883+0000","last_fullsized":"2024-04-13T07:32:39.016883+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:19:16.706609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010342+0000","last_change":"2024-04-13T07:32:03.077038+0000","last_active":"2024-04-13T07:32:39.010342+0000","last_peered":"2024-04-13T07:32:39.010342+0000","last_clean":"2024-04-13T07:32:39.010342+0000","last_became_active":"2024-04-13T07:32:03.076284+0000","last_became_peered":"2024-04-13T07:32:03.076284+0000","last_unstale":"2024-04-13T07:32:39.010342+0000","last_undegraded":"2024-04-13T07:32:39.010342+0000","last_fullsized":"2024-04-13T07:32:39.010342+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:01:47.340845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023330+0000","last_change":"2024-04-13T07:30:21.471551+0000","last_active":"2024-04-13T07:32:39.023330+0000","last_peered":"2024-04-13T07:32:39.023330+0000","last_clean":"2024-04-13T07:32:39.023330+0000","last_became_active":"2024-04-13T07:30:21.470542+0000","last_became_peered":"2024-04-13T07:30:21.470542+0000","last_unstale":"2024-04-13T07:32:39.023330+0000","last_undegraded":"2024-04-13T07:32:39.023330+0000","last_fullsized":"2024-04-13T07:32:39.023330+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:54:43.387491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013383+0000","last_change":"2024-04-13T07:32:04.418369+0000","last_active":"2024-04-13T07:32:39.013383+0000","last_peered":"2024-04-13T07:32:39.013383+0000","last_clean":"2024-04-13T07:32:39.013383+0000","last_became_active":"2024-04-13T07:32:04.418037+0000","last_became_peered":"2024-04-13T07:32:04.418037+0000","last_unstale":"2024-04-13T07:32:39.013383+0000","last_undegraded":"2024-04-13T07:32:39.013383+0000","last_fullsized":"2024-04-13T07:32:39.013383+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:54:59.849559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022408+0000","last_change":"2024-04-13T07:32:03.071040+0000","last_active":"2024-04-13T07:32:39.022408+0000","last_peered":"2024-04-13T07:32:39.022408+0000","last_clean":"2024-04-13T07:32:39.022408+0000","last_became_active":"2024-04-13T07:32:03.069086+0000","last_became_peered":"2024-04-13T07:32:03.069086+0000","last_unstale":"2024-04-13T07:32:39.022408+0000","last_undegraded":"2024-04-13T07:32:39.022408+0000","last_fullsized":"2024-04-13T07:32:39.022408+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:12:41.759089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021951+0000","last_change":"2024-04-13T07:30:21.469386+0000","last_active":"2024-04-13T07:32:39.021951+0000","last_peered":"2024-04-13T07:32:39.021951+0000","last_clean":"2024-04-13T07:32:39.021951+0000","last_became_active":"2024-04-13T07:30:21.468314+0000","last_became_peered":"2024-04-13T07:30:21.468314+0000","last_unstale":"2024-04-13T07:32:39.021951+0000","last_undegraded":"2024-04-13T07:32:39.021951+0000","last_fullsized":"2024-04-13T07:32:39.021951+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:51:04.281850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.011514+0000","last_change":"2024-04-13T07:32:04.374209+0000","last_active":"2024-04-13T07:32:39.011514+0000","last_peered":"2024-04-13T07:32:39.011514+0000","last_clean":"2024-04-13T07:32:39.011514+0000","last_became_active":"2024-04-13T07:32:04.373393+0000","last_became_peered":"2024-04-13T07:32:04.373393+0000","last_unstale":"2024-04-13T07:32:39.011514+0000","last_undegraded":"2024-04-13T07:32:39.011514+0000","last_fullsized":"2024-04-13T07:32:39.011514+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:57:46.913705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022449+0000","last_change":"2024-04-13T07:32:03.076043+0000","last_active":"2024-04-13T07:32:39.022449+0000","last_peered":"2024-04-13T07:32:39.022449+0000","last_clean":"2024-04-13T07:32:39.022449+0000","last_became_active":"2024-04-13T07:32:03.075193+0000","last_became_peered":"2024-04-13T07:32:03.075193+0000","last_unstale":"2024-04-13T07:32:39.022449+0000","last_undegraded":"2024-04-13T07:32:39.022449+0000","last_fullsized":"2024-04-13T07:32:39.022449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:07:57.007339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013164+0000","last_change":"2024-04-13T07:30:21.471173+0000","last_active":"2024-04-13T07:32:39.013164+0000","last_peered":"2024-04-13T07:32:39.013164+0000","last_clean":"2024-04-13T07:32:39.013164+0000","last_became_active":"2024-04-13T07:30:21.470193+0000","last_became_peered":"2024-04-13T07:30:21.470193+0000","last_unstale":"2024-04-13T07:32:39.013164+0000","last_undegraded":"2024-04-13T07:32:39.013164+0000","last_fullsized":"2024-04-13T07:32:39.013164+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:13:17.367790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-04-13T07:32:40.027446+0000","last_change":"2024-04-13T07:32:40.027446+0000","last_active":"2024-04-13T07:32:40.027446+0000","last_peered":"2024-04-13T07:32:40.027446+0000","last_clean":"2024-04-13T07:32:40.027446+0000","last_became_active":"2024-04-13T07:32:40.026430+0000","last_became_peered":"2024-04-13T07:32:40.026430+0000","last_unstale":"2024-04-13T07:32:40.027446+0000","last_undegraded":"2024-04-13T07:32:40.027446+0000","last_fullsized":"2024-04-13T07:32:40.027446+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:46:05.767985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.016554+0000","last_change":"2024-04-13T07:32:03.090544+0000","last_active":"2024-04-13T07:32:39.016554+0000","last_peered":"2024-04-13T07:32:39.016554+0000","last_clean":"2024-04-13T07:32:39.016554+0000","last_became_active":"2024-04-13T07:32:03.084852+0000","last_became_peered":"2024-04-13T07:32:03.084852+0000","last_unstale":"2024-04-13T07:32:39.016554+0000","last_undegraded":"2024-04-13T07:32:39.016554+0000","last_fullsized":"2024-04-13T07:32:39.016554+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:32:01.912602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.024037+0000","last_change":"2024-04-13T07:32:03.146367+0000","last_active":"2024-04-13T07:32:39.024037+0000","last_peered":"2024-04-13T07:32:39.024037+0000","last_clean":"2024-04-13T07:32:39.024037+0000","last_became_active":"2024-04-13T07:32:03.144578+0000","last_became_peered":"2024-04-13T07:32:03.144578+0000","last_unstale":"2024-04-13T07:32:39.024037+0000","last_undegraded":"2024-04-13T07:32:39.024037+0000","last_fullsized":"2024-04-13T07:32:39.024037+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:20:42.951481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023234+0000","last_change":"2024-04-13T07:32:03.050715+0000","last_active":"2024-04-13T07:32:39.023234+0000","last_peered":"2024-04-13T07:32:39.023234+0000","last_clean":"2024-04-13T07:32:39.023234+0000","last_became_active":"2024-04-13T07:32:03.049589+0000","last_became_peered":"2024-04-13T07:32:03.049589+0000","last_unstale":"2024-04-13T07:32:39.023234+0000","last_undegraded":"2024-04-13T07:32:39.023234+0000","last_fullsized":"2024-04-13T07:32:39.023234+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:12:21.131281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.024522+0000","last_change":"2024-04-13T07:32:03.138119+0000","last_active":"2024-04-13T07:32:39.024522+0000","last_peered":"2024-04-13T07:32:39.024522+0000","last_clean":"2024-04-13T07:32:39.024522+0000","last_became_active":"2024-04-13T07:32:03.137391+0000","last_became_peered":"2024-04-13T07:32:03.137391+0000","last_unstale":"2024-04-13T07:32:39.024522+0000","last_undegraded":"2024-04-13T07:32:39.024522+0000","last_fullsized":"2024-04-13T07:32:39.024522+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:24:01.849939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148632+0000","last_change":"2024-04-13T07:32:03.072704+0000","last_active":"2024-04-13T07:32:39.148632+0000","last_peered":"2024-04-13T07:32:39.148632+0000","last_clean":"2024-04-13T07:32:39.148632+0000","last_became_active":"2024-04-13T07:32:03.071880+0000","last_became_peered":"2024-04-13T07:32:03.071880+0000","last_unstale":"2024-04-13T07:32:39.148632+0000","last_undegraded":"2024-04-13T07:32:39.148632+0000","last_fullsized":"2024-04-13T07:32:39.148632+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:33:54.606613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023633+0000","last_change":"2024-04-13T07:32:03.146200+0000","last_active":"2024-04-13T07:32:39.023633+0000","last_peered":"2024-04-13T07:32:39.023633+0000","last_clean":"2024-04-13T07:32:39.023633+0000","last_became_active":"2024-04-13T07:32:03.145450+0000","last_became_peered":"2024-04-13T07:32:03.145450+0000","last_unstale":"2024-04-13T07:32:39.023633+0000","last_undegraded":"2024-04-13T07:32:39.023633+0000","last_fullsized":"2024-04-13T07:32:39.023633+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:03:02.507834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017533+0000","last_change":"2024-04-13T07:32:03.082757+0000","last_active":"2024-04-13T07:32:39.017533+0000","last_peered":"2024-04-13T07:32:39.017533+0000","last_clean":"2024-04-13T07:32:39.017533+0000","last_became_active":"2024-04-13T07:32:03.081868+0000","last_became_peered":"2024-04-13T07:32:03.081868+0000","last_unstale":"2024-04-13T07:32:39.017533+0000","last_undegraded":"2024-04-13T07:32:39.017533+0000","last_fullsized":"2024-04-13T07:32:39.017533+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:51:59.170277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010075+0000","last_change":"2024-04-13T07:32:04.376133+0000","last_active":"2024-04-13T07:32:39.010075+0000","last_peered":"2024-04-13T07:32:39.010075+0000","last_clean":"2024-04-13T07:32:39.010075+0000","last_became_active":"2024-04-13T07:32:04.375185+0000","last_became_peered":"2024-04-13T07:32:04.375185+0000","last_unstale":"2024-04-13T07:32:39.010075+0000","last_undegraded":"2024-04-13T07:32:39.010075+0000","last_fullsized":"2024-04-13T07:32:39.010075+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:31:10.129701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021757+0000","last_change":"2024-04-13T07:32:03.067875+0000","last_active":"2024-04-13T07:32:39.021757+0000","last_peered":"2024-04-13T07:32:39.021757+0000","last_clean":"2024-04-13T07:32:39.021757+0000","last_became_active":"2024-04-13T07:32:03.067001+0000","last_became_peered":"2024-04-13T07:32:03.067001+0000","last_unstale":"2024-04-13T07:32:39.021757+0000","last_undegraded":"2024-04-13T07:32:39.021757+0000","last_fullsized":"2024-04-13T07:32:39.021757+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:37:50.584428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148280+0000","last_change":"2024-04-13T07:32:04.423046+0000","last_active":"2024-04-13T07:32:39.148280+0000","last_peered":"2024-04-13T07:32:39.148280+0000","last_clean":"2024-04-13T07:32:39.148280+0000","last_became_active":"2024-04-13T07:32:04.421900+0000","last_became_peered":"2024-04-13T07:32:04.421900+0000","last_unstale":"2024-04-13T07:32:39.148280+0000","last_undegraded":"2024-04-13T07:32:39.148280+0000","last_fullsized":"2024-04-13T07:32:39.148280+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:21:04.585612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010306+0000","last_change":"2024-04-13T07:32:03.078696+0000","last_active":"2024-04-13T07:32:39.010306+0000","last_peered":"2024-04-13T07:32:39.010306+0000","last_clean":"2024-04-13T07:32:39.010306+0000","last_became_active":"2024-04-13T07:32:03.077798+0000","last_became_peered":"2024-04-13T07:32:03.077798+0000","last_unstale":"2024-04-13T07:32:39.010306+0000","last_undegraded":"2024-04-13T07:32:39.010306+0000","last_fullsized":"2024-04-13T07:32:39.010306+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:32:02.676056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022535+0000","last_change":"2024-04-13T07:32:03.143672+0000","last_active":"2024-04-13T07:32:39.022535+0000","last_peered":"2024-04-13T07:32:39.022535+0000","last_clean":"2024-04-13T07:32:39.022535+0000","last_became_active":"2024-04-13T07:32:03.143068+0000","last_became_peered":"2024-04-13T07:32:03.143068+0000","last_unstale":"2024-04-13T07:32:39.022535+0000","last_undegraded":"2024-04-13T07:32:39.022535+0000","last_fullsized":"2024-04-13T07:32:39.022535+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:47:11.391489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010283+0000","last_change":"2024-04-13T07:32:03.079072+0000","last_active":"2024-04-13T07:32:39.010283+0000","last_peered":"2024-04-13T07:32:39.010283+0000","last_clean":"2024-04-13T07:32:39.010283+0000","last_became_active":"2024-04-13T07:32:03.078769+0000","last_became_peered":"2024-04-13T07:32:03.078769+0000","last_unstale":"2024-04-13T07:32:39.010283+0000","last_undegraded":"2024-04-13T07:32:39.010283+0000","last_fullsized":"2024-04-13T07:32:39.010283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:25:42.191631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021493+0000","last_change":"2024-04-13T07:32:03.141831+0000","last_active":"2024-04-13T07:32:39.021493+0000","last_peered":"2024-04-13T07:32:39.021493+0000","last_clean":"2024-04-13T07:32:39.021493+0000","last_became_active":"2024-04-13T07:32:03.141053+0000","last_became_peered":"2024-04-13T07:32:03.141053+0000","last_unstale":"2024-04-13T07:32:39.021493+0000","last_undegraded":"2024-04-13T07:32:39.021493+0000","last_fullsized":"2024-04-13T07:32:39.021493+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:24:08.376151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010782+0000","last_change":"2024-04-13T07:32:03.093924+0000","last_active":"2024-04-13T07:32:39.010782+0000","last_peered":"2024-04-13T07:32:39.010782+0000","last_clean":"2024-04-13T07:32:39.010782+0000","last_became_active":"2024-04-13T07:32:03.093273+0000","last_became_peered":"2024-04-13T07:32:03.093273+0000","last_unstale":"2024-04-13T07:32:39.010782+0000","last_undegraded":"2024-04-13T07:32:39.010782+0000","last_fullsized":"2024-04-13T07:32:39.010782+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:53:00.895622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-04-13T07:32:40.240677+0000","last_change":"2024-04-13T07:32:40.240677+0000","last_active":"2024-04-13T07:32:40.240677+0000","last_peered":"2024-04-13T07:32:40.240677+0000","last_clean":"2024-04-13T07:32:40.240677+0000","last_became_active":"2024-04-13T07:32:40.240094+0000","last_became_peered":"2024-04-13T07:32:40.240094+0000","last_unstale":"2024-04-13T07:32:40.240677+0000","last_undegraded":"2024-04-13T07:32:40.240677+0000","last_fullsized":"2024-04-13T07:32:40.240677+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:49:33.573507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":32,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148825+0000","last_change":"2024-04-13T07:32:03.064053+0000","last_active":"2024-04-13T07:32:39.148825+0000","last_peered":"2024-04-13T07:32:39.148825+0000","last_clean":"2024-04-13T07:32:39.148825+0000","last_became_active":"2024-04-13T07:32:03.061879+0000","last_became_peered":"2024-04-13T07:32:03.061879+0000","last_unstale":"2024-04-13T07:32:39.148825+0000","last_undegraded":"2024-04-13T07:32:39.148825+0000","last_fullsized":"2024-04-13T07:32:39.148825+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:03:11.892760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009637+0000","last_change":"2024-04-13T07:32:04.423037+0000","last_active":"2024-04-13T07:32:39.009637+0000","last_peered":"2024-04-13T07:32:39.009637+0000","last_clean":"2024-04-13T07:32:39.009637+0000","last_became_active":"2024-04-13T07:32:04.422667+0000","last_became_peered":"2024-04-13T07:32:04.422667+0000","last_unstale":"2024-04-13T07:32:39.009637+0000","last_undegraded":"2024-04-13T07:32:39.009637+0000","last_fullsized":"2024-04-13T07:32:39.009637+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:07:35.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009276+0000","last_change":"2024-04-13T07:32:03.082422+0000","last_active":"2024-04-13T07:32:39.009276+0000","last_peered":"2024-04-13T07:32:39.009276+0000","last_clean":"2024-04-13T07:32:39.009276+0000","last_became_active":"2024-04-13T07:32:03.080059+0000","last_became_peered":"2024-04-13T07:32:03.080059+0000","last_unstale":"2024-04-13T07:32:39.009276+0000","last_undegraded":"2024-04-13T07:32:39.009276+0000","last_fullsized":"2024-04-13T07:32:39.009276+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:07:47.268352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022163+0000","last_change":"2024-04-13T07:32:03.143977+0000","last_active":"2024-04-13T07:32:39.022163+0000","last_peered":"2024-04-13T07:32:39.022163+0000","last_clean":"2024-04-13T07:32:39.022163+0000","last_became_active":"2024-04-13T07:32:03.142886+0000","last_became_peered":"2024-04-13T07:32:03.142886+0000","last_unstale":"2024-04-13T07:32:39.022163+0000","last_undegraded":"2024-04-13T07:32:39.022163+0000","last_fullsized":"2024-04-13T07:32:39.022163+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:21:20.364627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009638+0000","last_change":"2024-04-13T07:32:03.084887+0000","last_active":"2024-04-13T07:32:39.009638+0000","last_peered":"2024-04-13T07:32:39.009638+0000","last_clean":"2024-04-13T07:32:39.009638+0000","last_became_active":"2024-04-13T07:32:03.069100+0000","last_became_peered":"2024-04-13T07:32:03.069100+0000","last_unstale":"2024-04-13T07:32:39.009638+0000","last_undegraded":"2024-04-13T07:32:39.009638+0000","last_fullsized":"2024-04-13T07:32:39.009638+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:10:11.856147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017427+0000","last_change":"2024-04-13T07:32:04.422844+0000","last_active":"2024-04-13T07:32:39.017427+0000","last_peered":"2024-04-13T07:32:39.017427+0000","last_clean":"2024-04-13T07:32:39.017427+0000","last_became_active":"2024-04-13T07:32:04.422538+0000","last_became_peered":"2024-04-13T07:32:04.422538+0000","last_unstale":"2024-04-13T07:32:39.017427+0000","last_undegraded":"2024-04-13T07:32:39.017427+0000","last_fullsized":"2024-04-13T07:32:39.017427+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:33:44.840955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009835+0000","last_change":"2024-04-13T07:32:03.081497+0000","last_active":"2024-04-13T07:32:39.009835+0000","last_peered":"2024-04-13T07:32:39.009835+0000","last_clean":"2024-04-13T07:32:39.009835+0000","last_became_active":"2024-04-13T07:32:03.080871+0000","last_became_peered":"2024-04-13T07:32:03.080871+0000","last_unstale":"2024-04-13T07:32:39.009835+0000","last_undegraded":"2024-04-13T07:32:39.009835+0000","last_fullsized":"2024-04-13T07:32:39.009835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:52:03.244799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022634+0000","last_change":"2024-04-13T07:32:03.144209+0000","last_active":"2024-04-13T07:32:39.022634+0000","last_peered":"2024-04-13T07:32:39.022634+0000","last_clean":"2024-04-13T07:32:39.022634+0000","last_became_active":"2024-04-13T07:32:03.141741+0000","last_became_peered":"2024-04-13T07:32:03.141741+0000","last_unstale":"2024-04-13T07:32:39.022634+0000","last_undegraded":"2024-04-13T07:32:39.022634+0000","last_fullsized":"2024-04-13T07:32:39.022634+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:36:52.296757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"70'7","reported_seq":29,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023835+0000","last_change":"2024-04-13T07:32:03.074002+0000","last_active":"2024-04-13T07:32:39.023835+0000","last_peered":"2024-04-13T07:32:39.023835+0000","last_clean":"2024-04-13T07:32:39.023835+0000","last_became_active":"2024-04-13T07:32:03.073163+0000","last_became_peered":"2024-04-13T07:32:03.073163+0000","last_unstale":"2024-04-13T07:32:39.023835+0000","last_undegraded":"2024-04-13T07:32:39.023835+0000","last_fullsized":"2024-04-13T07:32:39.023835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:50:04.054552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.011309+0000","last_change":"2024-04-13T07:32:04.418476+0000","last_active":"2024-04-13T07:32:39.011309+0000","last_peered":"2024-04-13T07:32:39.011309+0000","last_clean":"2024-04-13T07:32:39.011309+0000","last_became_active":"2024-04-13T07:32:04.418126+0000","last_became_peered":"2024-04-13T07:32:04.418126+0000","last_unstale":"2024-04-13T07:32:39.011309+0000","last_undegraded":"2024-04-13T07:32:39.011309+0000","last_fullsized":"2024-04-13T07:32:39.011309+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:24:25.424210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":22,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009904+0000","last_change":"2024-04-13T07:32:03.087811+0000","last_active":"2024-04-13T07:32:39.009904+0000","last_peered":"2024-04-13T07:32:39.009904+0000","last_clean":"2024-04-13T07:32:39.009904+0000","last_became_active":"2024-04-13T07:32:03.087293+0000","last_became_peered":"2024-04-13T07:32:03.087293+0000","last_unstale":"2024-04-13T07:32:39.009904+0000","last_undegraded":"2024-04-13T07:32:39.009904+0000","last_fullsized":"2024-04-13T07:32:39.009904+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:07:16.065771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017771+0000","last_change":"2024-04-13T07:32:04.423028+0000","last_active":"2024-04-13T07:32:39.017771+0000","last_peered":"2024-04-13T07:32:39.017771+0000","last_clean":"2024-04-13T07:32:39.017771+0000","last_became_active":"2024-04-13T07:32:04.422663+0000","last_became_peered":"2024-04-13T07:32:04.422663+0000","last_unstale":"2024-04-13T07:32:39.017771+0000","last_undegraded":"2024-04-13T07:32:39.017771+0000","last_fullsized":"2024-04-13T07:32:39.017771+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:37:23.228475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021870+0000","last_change":"2024-04-13T07:32:03.068639+0000","last_active":"2024-04-13T07:32:39.021870+0000","last_peered":"2024-04-13T07:32:39.021870+0000","last_clean":"2024-04-13T07:32:39.021870+0000","last_became_active":"2024-04-13T07:32:03.067401+0000","last_became_peered":"2024-04-13T07:32:03.067401+0000","last_unstale":"2024-04-13T07:32:39.021870+0000","last_undegraded":"2024-04-13T07:32:39.021870+0000","last_fullsized":"2024-04-13T07:32:39.021870+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:51:09.165741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.016703+0000","last_change":"2024-04-13T07:32:04.104373+0000","last_active":"2024-04-13T07:32:39.016703+0000","last_peered":"2024-04-13T07:32:39.016703+0000","last_clean":"2024-04-13T07:32:39.016703+0000","last_became_active":"2024-04-13T07:32:04.103428+0000","last_became_peered":"2024-04-13T07:32:04.103428+0000","last_unstale":"2024-04-13T07:32:39.016703+0000","last_undegraded":"2024-04-13T07:32:39.016703+0000","last_fullsized":"2024-04-13T07:32:39.016703+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:39:29.039199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013235+0000","last_change":"2024-04-13T07:32:03.089122+0000","last_active":"2024-04-13T07:32:39.013235+0000","last_peered":"2024-04-13T07:32:39.013235+0000","last_clean":"2024-04-13T07:32:39.013235+0000","last_became_active":"2024-04-13T07:32:03.087087+0000","last_became_peered":"2024-04-13T07:32:03.087087+0000","last_unstale":"2024-04-13T07:32:39.013235+0000","last_undegraded":"2024-04-13T07:32:39.013235+0000","last_fullsized":"2024-04-13T07:32:39.013235+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:15:12.555631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148826+0000","last_change":"2024-04-13T07:32:04.102324+0000","last_active":"2024-04-13T07:32:39.148826+0000","last_peered":"2024-04-13T07:32:39.148826+0000","last_clean":"2024-04-13T07:32:39.148826+0000","last_became_active":"2024-04-13T07:32:04.101488+0000","last_became_peered":"2024-04-13T07:32:04.101488+0000","last_unstale":"2024-04-13T07:32:39.148826+0000","last_undegraded":"2024-04-13T07:32:39.148826+0000","last_fullsized":"2024-04-13T07:32:39.148826+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:32:06.780234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017007+0000","last_change":"2024-04-13T07:32:03.084186+0000","last_active":"2024-04-13T07:32:39.017007+0000","last_peered":"2024-04-13T07:32:39.017007+0000","last_clean":"2024-04-13T07:32:39.017007+0000","last_became_active":"2024-04-13T07:32:03.080930+0000","last_became_peered":"2024-04-13T07:32:03.080930+0000","last_unstale":"2024-04-13T07:32:39.017007+0000","last_undegraded":"2024-04-13T07:32:39.017007+0000","last_fullsized":"2024-04-13T07:32:39.017007+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:22:54.864468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.011288+0000","last_change":"2024-04-13T07:32:04.423862+0000","last_active":"2024-04-13T07:32:39.011288+0000","last_peered":"2024-04-13T07:32:39.011288+0000","last_clean":"2024-04-13T07:32:39.011288+0000","last_became_active":"2024-04-13T07:32:04.423514+0000","last_became_peered":"2024-04-13T07:32:04.423514+0000","last_unstale":"2024-04-13T07:32:39.011288+0000","last_undegraded":"2024-04-13T07:32:39.011288+0000","last_fullsized":"2024-04-13T07:32:39.011288+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:55:46.044341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010738+0000","last_change":"2024-04-13T07:32:03.081098+0000","last_active":"2024-04-13T07:32:39.010738+0000","last_peered":"2024-04-13T07:32:39.010738+0000","last_clean":"2024-04-13T07:32:39.010738+0000","last_became_active":"2024-04-13T07:32:03.080593+0000","last_became_peered":"2024-04-13T07:32:03.080593+0000","last_unstale":"2024-04-13T07:32:39.010738+0000","last_undegraded":"2024-04-13T07:32:39.010738+0000","last_fullsized":"2024-04-13T07:32:39.010738+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:07:33.886771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-04-13T07:32:40.032333+0000","last_change":"2024-04-13T07:32:40.032333+0000","last_active":"2024-04-13T07:32:40.032333+0000","last_peered":"2024-04-13T07:32:40.032333+0000","last_clean":"2024-04-13T07:32:40.032333+0000","last_became_active":"2024-04-13T07:32:40.031380+0000","last_became_peered":"2024-04-13T07:32:40.031380+0000","last_unstale":"2024-04-13T07:32:40.032333+0000","last_undegraded":"2024-04-13T07:32:40.032333+0000","last_fullsized":"2024-04-13T07:32:40.032333+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:00:32.658370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022818+0000","last_change":"2024-04-13T07:32:03.060936+0000","last_active":"2024-04-13T07:32:39.022818+0000","last_peered":"2024-04-13T07:32:39.022818+0000","last_clean":"2024-04-13T07:32:39.022818+0000","last_became_active":"2024-04-13T07:32:03.059737+0000","last_became_peered":"2024-04-13T07:32:03.059737+0000","last_unstale":"2024-04-13T07:32:39.022818+0000","last_undegraded":"2024-04-13T07:32:39.022818+0000","last_fullsized":"2024-04-13T07:32:39.022818+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:44:10.168724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":77,"num_read_kb":80,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201340,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32452,"kb_used_data":1332,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712700,"statfs":{"total":95995035648,"available":95961804800,"internally_reserved":0,"allocated":1363968,"data_stored":916725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.88600000000000001,"15min":0.878},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.264,"5min":1.3,"15min":1.3},"last":0.96299999999999997},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.90700000000000003,"15min":0.92700000000000005},"min":{"1min":0.59599999999999997,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.548,"5min":1.548,"15min":1.548},"last":0.71599999999999997}]},{"osd":1,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.88500000000000001,"15min":0.88},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3149999999999999,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":1.0960000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.96199999999999997,"15min":0.95999999999999996},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.391,"5min":1.391,"15min":1.391},"last":1.0429999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.92600000000000005,"15min":0.92300000000000004},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.397,"5min":1.397,"15min":1.397},"last":1.0049999999999999},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.999,"15min":1.0129999999999999},"min":{"1min":0.56899999999999995,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.2549999999999999,"5min":1.369,"15min":1.369},"last":0.77500000000000002}]},{"osd":3,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":0.94899999999999995,"15min":0.94499999999999995},"min":{"1min":0.55600000000000005,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.304,"5min":1.401,"15min":1.401},"last":0.81200000000000006},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.0529999999999999,"15min":1.0449999999999999},"min":{"1min":0.70199999999999996,"5min":0.501,"15min":0.501},"max":{"1min":1.3460000000000001,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.1950000000000001}]},{"osd":4,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.91200000000000003,"15min":0.88500000000000001},"min":{"1min":0.45600000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":0.873},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.95199999999999996,"15min":0.93500000000000005},"min":{"1min":0.502,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":0.91300000000000003}]},{"osd":5,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.028,"15min":1.006},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.407,"5min":1.492,"15min":1.492},"last":1.29},{"interface":"front","average":{"1min":1.147,"5min":1.0669999999999999,"15min":1.0529999999999999},"min":{"1min":0.59599999999999997,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.474,"5min":1.548,"15min":1.548},"last":1.1559999999999999}]},{"osd":6,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.109,"15min":1.1020000000000001},"min":{"1min":0.64100000000000001,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.337,"5min":1.536,"15min":1.536},"last":1.236},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0900000000000001,"15min":1.0980000000000001},"min":{"1min":0.68899999999999995,"5min":0.434,"15min":0.434},"max":{"1min":1.4590000000000001,"5min":1.633,"15min":1.633},"last":1.0800000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462982,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1343488,"data_stored":915975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.90000000000000002,"15min":0.85899999999999999},"min":{"1min":0.56899999999999995,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.145,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":0.77100000000000002},{"interface":"front","average":{"1min":0.995,"5min":0.93400000000000005,"15min":0.89600000000000002},"min":{"1min":0.67000000000000004,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2310000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.181}]},{"osd":1,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.92800000000000005,"15min":0.92400000000000004},"min":{"1min":0.60999999999999999,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.444,"5min":1.444,"15min":1.444},"last":0.90200000000000002},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.92200000000000004,"15min":0.93000000000000005},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.167,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.93700000000000006}]},{"osd":2,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.95499999999999996,"15min":0.91300000000000003},"min":{"1min":0.57099999999999995,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.2270000000000001,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":1.032},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.97899999999999998,"15min":1.004},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":0.97899999999999998}]},{"osd":3,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.95399999999999996,"15min":0.93000000000000005},"min":{"1min":0.58799999999999997,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.3169999999999999,"5min":1.45,"15min":1.45},"last":1.0680000000000001},{"interface":"front","average":{"1min":1.046,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.70299999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.302,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.139}]},{"osd":4,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":0.92000000000000004,"15min":0.90200000000000002},"min":{"1min":0.745,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":1.107},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.94399999999999995,"15min":0.87},"min":{"1min":0.68300000000000005,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.333,"5min":1.5,"15min":1.5},"last":1.29}]},{"osd":5,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.99199999999999999,"15min":0.92600000000000005},"min":{"1min":0.749,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.286,"5min":1.423,"15min":1.423},"last":1.26},{"interface":"front","average":{"1min":1.117,"5min":1.038,"15min":0.997},"min":{"1min":0.78200000000000003,"5min":0.38,"15min":0.38},"max":{"1min":1.349,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":1.3680000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:32:43 2024","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":1.135,"15min":1.147},"min":{"1min":0.80100000000000005,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.5149999999999999,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.169,"5min":1.1299999999999999,"15min":1.1419999999999999},"min":{"1min":0.79600000000000004,"5min":0.44,"15min":0.44},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":1.335}]}]},{"osd":1,"up_from":13,"seq":55834574969,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":741376,"data_stored":324577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.65900000000000003,"15min":0.57699999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.0980000000000001,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.85099999999999998,"15min":0.68899999999999995},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.3839999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.252}]},{"osd":2,"last update":"Sat Apr 13 07:32:32 2024","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.70499999999999996,"15min":0.65600000000000003},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.66600000000000004},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.84499999999999997,"15min":0.74099999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.27900000000000003},"max":{"1min":1.26,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.72999999999999998}]},{"osd":3,"last update":"Sat Apr 13 07:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.86299999999999999,"15min":0.79200000000000004},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":1.22,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.996,"15min":0.90500000000000003},"min":{"1min":0.39500000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.357,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":1.099}]},{"osd":4,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91800000000000004,"15min":0.91700000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.302,"5min":1.349,"15min":1.349},"last":1.133},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0089999999999999,"15min":0.95299999999999996},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.3500000000000001,"5min":1.998,"15min":1.998},"last":0.94899999999999995}]},{"osd":5,"last update":"Sat Apr 13 07:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0640000000000001,"15min":1.044},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.4530000000000001,"5min":2.0449999999999999,"15min":2.0449999999999999},"last":1.034},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.004,"15min":0.99199999999999999},"min":{"1min":0.61799999999999999,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":0.85499999999999998}]},{"osd":6,"last update":"Sat Apr 13 07:32:56 2024","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0349999999999999,"15min":1.026},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.6419999999999999,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":1.0669999999999999},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.157,"15min":1.1579999999999999},"min":{"1min":0.71199999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.954,"5min":1.954,"15min":1.954},"last":1.198}]},{"osd":7,"last update":"Sat Apr 13 07:32:51 2024","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.1259999999999999,"15min":1.1439999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":0.99099999999999999},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.1459999999999999,"15min":1.1759999999999999},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":2.214,"5min":2.214,"15min":2.214},"last":1.1539999999999999}]}]},{"osd":0,"up_from":8,"seq":34359738500,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32432,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712720,"statfs":{"total":95995035648,"available":95961825280,"internally_reserved":0,"allocated":1343488,"data_stored":915549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 13 07:32:50 2024","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.69199999999999995,"15min":0.58899999999999997},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.26500000000000001},"max":{"1min":1.036,"5min":1.1140000000000001,"15min":1.1910000000000001},"last":0.93000000000000005},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.78800000000000003,"15min":0.64700000000000002},"min":{"1min":0.5,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0109999999999999,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.86899999999999999}]},{"osd":2,"last update":"Sat Apr 13 07:32:30 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.76200000000000001,"15min":0.69899999999999995},"min":{"1min":0.39800000000000002,"5min":0.311,"15min":0.311},"max":{"1min":1.0960000000000001,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.78100000000000003,"15min":0.70399999999999996},"min":{"1min":0.442,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.0800000000000001,"5min":1.1919999999999999,"15min":1.2},"last":0.63600000000000001}]},{"osd":3,"last update":"Sat Apr 13 07:32:27 2024","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.78600000000000003,"15min":0.76100000000000001},"min":{"1min":0.41599999999999998,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.2589999999999999,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.79700000000000004},{"interface":"front","average":{"1min":0.83399999999999996,"5min":0.84099999999999997,"15min":0.82199999999999995},"min":{"1min":0.52000000000000002,"5min":0.378,"15min":0.378},"max":{"1min":1.175,"5min":1.2050000000000001,"15min":1.2629999999999999},"last":0.75800000000000001}]},{"osd":4,"last update":"Sat Apr 13 07:33:11 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.99299999999999999,"15min":0.95399999999999996},"min":{"1min":0.59999999999999998,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.2110000000000001,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.84499999999999997},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.0600000000000001,"15min":1.014},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.3320000000000001,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":1.044}]},{"osd":5,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.0720000000000001,"15min":1.024},"min":{"1min":0.60199999999999998,"5min":0.501,"15min":0.501},"max":{"1min":1.55,"5min":1.748,"15min":1.748},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.044,"5min":1.0600000000000001,"15min":1.008},"min":{"1min":0.626,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.96399999999999997}]},{"osd":6,"last update":"Sat Apr 13 07:32:57 2024","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.135,"15min":1.1659999999999999},"min":{"1min":0.64000000000000001,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.5569999999999999,"5min":1.621,"15min":1.621},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.054,"5min":1.1100000000000001,"15min":1.0960000000000001},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.494,"5min":1.659,"15min":1.659},"last":0.90900000000000003}]},{"osd":7,"last update":"Sat Apr 13 07:32:50 2024","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.181,"15min":1.218},"min":{"1min":0.67600000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.879,"5min":1.879,"15min":1.879},"last":1.1379999999999999},{"interface":"front","average":{"1min":1.246,"5min":1.23,"15min":1.24},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.7909999999999999,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.3029999999999999}]}]},{"osd":2,"up_from":18,"seq":77309411439,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":749568,"data_stored":325193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.72399999999999998,"15min":0.627},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0600000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.77000000000000002,"15min":0.68300000000000005},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.35999999999999999},"max":{"1min":1.0089999999999999,"5min":1.296,"15min":1.296},"last":1.01}]},{"osd":1,"last update":"Sat Apr 13 07:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.78900000000000003,"15min":0.69699999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33200000000000002},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.82799999999999996},{"interface":"front","average":{"1min":0.873,"5min":0.84099999999999997,"15min":0.72999999999999998},"min":{"1min":0.44400000000000001,"5min":0.309,"15min":0.28899999999999998},"max":{"1min":1.748,"5min":1.748,"15min":1.748},"last":0.79300000000000004}]},{"osd":3,"last update":"Sat Apr 13 07:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.88500000000000001,"15min":0.80600000000000005},"min":{"1min":0.46600000000000003,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.1819999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.89000000000000001},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.89900000000000002,"15min":0.84299999999999997},"min":{"1min":0.53700000000000003,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":1.091}]},{"osd":4,"last update":"Sat Apr 13 07:33:22 2024","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91000000000000003,"15min":0.91400000000000003},"min":{"1min":0.628,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1200000000000001,"5min":1.341,"15min":1.341},"last":1.0509999999999999},{"interface":"front","average":{"1min":1.04,"5min":1.0609999999999999,"15min":1.036},"min":{"1min":0.57999999999999996,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.4279999999999999,"5min":1.52,"15min":1.52},"last":1.3899999999999999}]},{"osd":5,"last update":"Sat Apr 13 07:33:01 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0469999999999999,"15min":1.0569999999999999},"min":{"1min":0.70299999999999996,"5min":0.498,"15min":0.498},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":1.151},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.95399999999999996,"15min":0.96799999999999997},"min":{"1min":0.65700000000000003,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.393,"5min":1.456,"15min":1.4850000000000001},"last":0.95299999999999996}]},{"osd":6,"last update":"Sat Apr 13 07:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.123,"15min":1.1699999999999999},"min":{"1min":0.61399999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.554,"5min":1.788,"15min":1.788},"last":1.3140000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.1279999999999999,"15min":1.135},"min":{"1min":0.65000000000000002,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.9199999999999999,"5min":1.9199999999999999,"15min":1.9199999999999999},"last":1.196}]},{"osd":7,"last update":"Sat Apr 13 07:32:45 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.1439999999999999,"15min":1.1279999999999999},"min":{"1min":0.63100000000000001,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.6419999999999999,"5min":1.651,"15min":1.651},"last":1.444},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"min":{"1min":0.72199999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":2.008,"5min":2.2130000000000001,"15min":2.2130000000000001},"last":1.4159999999999999}]}]},{"osd":3,"up_from":26,"seq":111669149797,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":745472,"data_stored":324667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:32:25 2024","interfaces":[{"interface":"back","average":{"1min":0.67700000000000005,"5min":0.67100000000000004,"15min":0.66500000000000004},"min":{"1min":0.34899999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.0369999999999999,"5min":1.196,"15min":1.196},"last":0.74199999999999999},{"interface":"front","average":{"1min":0.77500000000000002,"5min":0.73799999999999999,"15min":0.76200000000000001},"min":{"1min":0.377,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.056,"5min":1.1599999999999999,"15min":1.206},"last":0.90700000000000003}]},{"osd":1,"last update":"Sat Apr 13 07:32:25 2024","interfaces":[{"interface":"back","average":{"1min":0.70999999999999996,"5min":0.71599999999999997,"15min":0.73299999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.093,"5min":1.093,"15min":1.093},"last":0.79300000000000004},{"interface":"front","average":{"1min":0.80200000000000005,"5min":0.78800000000000003,"15min":0.78700000000000003},"min":{"1min":0.315,"5min":0.314,"15min":0.314},"max":{"1min":1.206,"5min":1.206,"15min":1.254},"last":1.054}]},{"osd":2,"last update":"Sat Apr 13 07:32:25 2024","interfaces":[{"interface":"back","average":{"1min":0.82299999999999995,"5min":0.83299999999999996,"15min":0.83499999999999996},"min":{"1min":0.39800000000000002,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.01,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":1.1679999999999999},{"interface":"front","average":{"1min":0.94899999999999995,"5min":0.86099999999999999,"15min":0.82399999999999995},"min":{"1min":0.30499999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.345,"5min":1.345,"15min":1.345},"last":0.82699999999999996}]},{"osd":4,"last update":"Sat Apr 13 07:33:12 2024","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.92900000000000005,"15min":0.91400000000000003},"min":{"1min":0.61799999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.242,"5min":1.347,"15min":1.347},"last":1.1240000000000001},{"interface":"front","average":{"1min":1.012,"5min":1.0069999999999999,"15min":0.96399999999999997},"min":{"1min":0.82799999999999996,"5min":0.495,"15min":0.495},"max":{"1min":1.367,"5min":1.367,"15min":1.389},"last":1.093}]},{"osd":5,"last update":"Sat Apr 13 07:33:09 2024","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.046,"15min":1.0349999999999999},"min":{"1min":0.76600000000000001,"5min":0.438,"15min":0.438},"max":{"1min":1.274,"5min":1.46,"15min":1.46},"last":1.3120000000000001},{"interface":"front","average":{"1min":1.091,"5min":1.0669999999999999,"15min":1.0589999999999999},"min":{"1min":0.82099999999999995,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.1899999999999999}]},{"osd":6,"last update":"Sat Apr 13 07:32:57 2024","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.081,"15min":1.0700000000000001},"min":{"1min":0.92500000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3999999999999999,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.23},{"interface":"front","average":{"1min":1.177,"5min":1.0980000000000001,"15min":1.0680000000000001},"min":{"1min":0.93300000000000005,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.3540000000000001,"5min":1.536,"15min":1.536},"last":1.2549999999999999}]},{"osd":7,"last update":"Sat Apr 13 07:32:47 2024","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.127,"15min":1.0820000000000001},"min":{"1min":1.0109999999999999,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":1.153},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.1459999999999999,"15min":1.1299999999999999},"min":{"1min":0.88800000000000001,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.4470000000000001,"5min":1.611,"15min":1.611},"last":1.3560000000000001}]}]},{"osd":4,"up_from":31,"seq":133143986267,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31868,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713284,"statfs":{"total":95995035648,"available":95962402816,"internally_reserved":0,"allocated":765952,"data_stored":326287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.84899999999999998,"15min":0.86399999999999999},"min":{"1min":0.624,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.1180000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":1.222},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.95999999999999996,"15min":0.90900000000000003},"min":{"1min":0.54500000000000004,"5min":0.432,"15min":0.432},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.175}]},{"osd":1,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.91400000000000003,"15min":0.89500000000000002},"min":{"1min":0.627,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.24,"5min":1.333,"15min":1.6819999999999999},"last":1.0209999999999999},{"interface":"front","average":{"1min":0.876,"5min":0.89300000000000002,"15min":0.97699999999999998},"min":{"1min":0.54700000000000004,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.2070000000000001,"5min":1.2929999999999999,"15min":1.7190000000000001},"last":0.85099999999999998}]},{"osd":2,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.95799999999999996,"15min":0.91200000000000003},"min":{"1min":0.55400000000000005,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1910000000000001,"5min":1.2729999999999999,"15min":1.413},"last":1.1399999999999999},{"interface":"front","average":{"1min":1.028,"5min":0.96599999999999997,"15min":0.90500000000000003},"min":{"1min":0.67000000000000004,"5min":0.502,"15min":0.502},"max":{"1min":1.353,"5min":1.48,"15min":1.48},"last":0.88600000000000001}]},{"osd":3,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.97699999999999998,"15min":0.93000000000000005},"min":{"1min":0.64800000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.258,"5min":1.4910000000000001,"15min":1.7849999999999999},"last":0.95599999999999996},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.98299999999999998,"15min":0.92500000000000004},"min":{"1min":0.72999999999999998,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.3140000000000001,"5min":1.524,"15min":1.524},"last":0.96999999999999997}]},{"osd":5,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.92400000000000004,"15min":0.81000000000000005},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2310000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.999},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.94199999999999995,"15min":0.75600000000000001},"min":{"1min":0.46500000000000002,"5min":0.35299999999999998,"15min":0.28000000000000003},"max":{"1min":1.371,"5min":1.76,"15min":1.76},"last":0.81899999999999995}]},{"osd":6,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1,"15min":0.95399999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.2929999999999999,"5min":1.675,"15min":1.675},"last":1.081},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.0249999999999999,"15min":0.999},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.337,"5min":1.734,"15min":1.734},"last":1.1100000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:32:51 2024","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.034,"15min":0.99199999999999999},"min":{"1min":0.40600000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.3520000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":0.91700000000000004},{"interface":"front","average":{"1min":1.117,"5min":1.0209999999999999,"15min":0.96899999999999997},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.5569999999999999,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":1.0349999999999999}]}]},{"osd":5,"up_from":39,"seq":167503724625,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":745472,"data_stored":324898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.92100000000000004,"15min":0.90000000000000002},"min":{"1min":0.70399999999999996,"5min":0.435,"15min":0.435},"max":{"1min":1.4410000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":1.0029999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.93100000000000005,"15min":0.89100000000000001},"min":{"1min":0.629,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1950000000000001,"5min":1.266,"15min":1.266},"last":0.84799999999999998}]},{"osd":1,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97399999999999998,"15min":0.91800000000000004},"min":{"1min":0.67500000000000004,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3859999999999999,"5min":1.524,"15min":1.524},"last":1.085},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.92600000000000005,"15min":0.92200000000000004},"min":{"1min":0.70899999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.2350000000000001,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":0.90700000000000003}]},{"osd":2,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.002,"15min":0.92400000000000004},"min":{"1min":0.71599999999999997,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.2589999999999999,"5min":1.343,"15min":1.343},"last":0.96299999999999997},{"interface":"front","average":{"1min":1.002,"5min":0.98899999999999999,"15min":0.95499999999999996},"min":{"1min":0.65700000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":0.94199999999999995}]},{"osd":3,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.0049999999999999,"15min":0.95299999999999996},"min":{"1min":0.77400000000000002,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":1.2370000000000001},{"interface":"front","average":{"1min":1.069,"5min":1.038,"15min":0.98799999999999999},"min":{"1min":0.746,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4450000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":0.878}]},{"osd":4,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.89100000000000001,"15min":0.79900000000000004},"min":{"1min":0.57399999999999995,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.3129999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.997,"15min":0.85099999999999998},"min":{"1min":0.68999999999999995,"5min":0.318,"15min":0.318},"max":{"1min":1.448,"5min":1.7350000000000001,"15min":1.7350000000000001},"last":0.80200000000000005}]},{"osd":6,"last update":"Sat Apr 13 07:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.0249999999999999,"15min":0.85899999999999999},"min":{"1min":0.64700000000000002,"5min":0.37,"15min":0.37},"max":{"1min":1.5069999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":1.157},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0529999999999999,"15min":0.85699999999999998},"min":{"1min":0.71699999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.55,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":0.67900000000000005}]},{"osd":7,"last update":"Sat Apr 13 07:32:49 2024","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.038,"15min":0.95599999999999996},"min":{"1min":0.625,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.3959999999999999,"5min":1.454,"15min":1.454},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.1539999999999999,"15min":1.1000000000000001},"min":{"1min":0.74299999999999999,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.0489999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-13T07:33:27.347 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-13T07:33:27.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:27 smithi203 bash[23480]: audit 2024-04-13T07:33:26.712247+0000 mgr.y (mgr.14373) 154 : audit 0 from='client.24614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:33:27.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:27 smithi089 bash[31522]: audit 2024-04-13T07:33:26.712247+0000 mgr.y (mgr.14373) 154 : audit 0 from='client.24614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:33:27.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:27 smithi089 bash[21353]: audit 2024-04-13T07:33:26.712247+0000 mgr.y (mgr.14373) 154 : audit 0 from='client.24614 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:33:28.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:28 smithi203 bash[23480]: cluster 2024-04-13T07:33:26.921223+0000 mgr.y (mgr.14373) 155 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:28.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:28 smithi089 bash[21353]: cluster 2024-04-13T07:33:26.921223+0000 mgr.y (mgr.14373) 155 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:28.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:28 smithi089 bash[31522]: cluster 2024-04-13T07:33:26.921223+0000 mgr.y (mgr.14373) 155 : cluster 0 pgmap v129: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:30.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:30 smithi203 bash[23480]: cluster 2024-04-13T07:33:28.921839+0000 mgr.y (mgr.14373) 156 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:30.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:30 smithi089 bash[31522]: cluster 2024-04-13T07:33:28.921839+0000 mgr.y (mgr.14373) 156 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:30.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:30 smithi089 bash[21353]: cluster 2024-04-13T07:33:28.921839+0000 mgr.y (mgr.14373) 156 : cluster 0 pgmap v130: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:32.087 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:32.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:32 smithi203 bash[23480]: cluster 2024-04-13T07:33:30.922930+0000 mgr.y (mgr.14373) 157 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:32.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:32 smithi089 bash[21353]: cluster 2024-04-13T07:33:30.922930+0000 mgr.y (mgr.14373) 157 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:32.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:32 smithi089 bash[31522]: cluster 2024-04-13T07:33:30.922930+0000 mgr.y (mgr.14373) 157 : cluster 0 pgmap v131: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:34.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:34 smithi203 bash[23480]: cluster 2024-04-13T07:33:32.923547+0000 mgr.y (mgr.14373) 158 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:34.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:34 smithi089 bash[31522]: cluster 2024-04-13T07:33:32.923547+0000 mgr.y (mgr.14373) 158 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:34.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:34 smithi089 bash[21353]: cluster 2024-04-13T07:33:32.923547+0000 mgr.y (mgr.14373) 158 : cluster 0 pgmap v132: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:34.609 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:33:34.610 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-04-13T07:33:35.172 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":132,"stamp":"2024-04-13T07:33:32.923216+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592730,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":289,"num_read_kb":386,"num_write":224,"num_write_kb":2430,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":173,"ondisk_log_size":173,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":236084,"kb_used_data":7616,"kb_used_omap":12,"kb_used_meta":228339,"kb_avail":749725132,"statfs":{"total":767960285184,"available":767718535168,"internally_reserved":0,"allocated":7798784,"data_stored":4373871,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":12743,"internal_metadata":233819705},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.005362"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021925+0000","last_change":"2024-04-13T07:32:03.142678+0000","last_active":"2024-04-13T07:32:39.021925+0000","last_peered":"2024-04-13T07:32:39.021925+0000","last_clean":"2024-04-13T07:32:39.021925+0000","last_became_active":"2024-04-13T07:32:03.141871+0000","last_became_peered":"2024-04-13T07:32:03.141871+0000","last_unstale":"2024-04-13T07:32:39.021925+0000","last_undegraded":"2024-04-13T07:32:39.021925+0000","last_fullsized":"2024-04-13T07:32:39.021925+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:10:35.700451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":23,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013869+0000","last_change":"2024-04-13T07:32:03.086097+0000","last_active":"2024-04-13T07:32:39.013869+0000","last_peered":"2024-04-13T07:32:39.013869+0000","last_clean":"2024-04-13T07:32:39.013869+0000","last_became_active":"2024-04-13T07:32:03.066927+0000","last_became_peered":"2024-04-13T07:32:03.066927+0000","last_unstale":"2024-04-13T07:32:39.013869+0000","last_undegraded":"2024-04-13T07:32:39.013869+0000","last_fullsized":"2024-04-13T07:32:39.013869+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:39:35.420226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":4,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017118+0000","last_change":"2024-04-13T07:32:04.418359+0000","last_active":"2024-04-13T07:32:39.017118+0000","last_peered":"2024-04-13T07:32:39.017118+0000","last_clean":"2024-04-13T07:32:39.017118+0000","last_became_active":"2024-04-13T07:32:04.418051+0000","last_became_peered":"2024-04-13T07:32:04.418051+0000","last_unstale":"2024-04-13T07:32:39.017118+0000","last_undegraded":"2024-04-13T07:32:39.017118+0000","last_fullsized":"2024-04-13T07:32:39.017118+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:54:44.282519+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.018012+0000","last_change":"2024-04-13T07:32:03.080811+0000","last_active":"2024-04-13T07:32:39.018012+0000","last_peered":"2024-04-13T07:32:39.018012+0000","last_clean":"2024-04-13T07:32:39.018012+0000","last_became_active":"2024-04-13T07:32:03.072598+0000","last_became_peered":"2024-04-13T07:32:03.072598+0000","last_unstale":"2024-04-13T07:32:39.018012+0000","last_undegraded":"2024-04-13T07:32:39.018012+0000","last_fullsized":"2024-04-13T07:32:39.018012+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:13:04.105633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023060+0000","last_change":"2024-04-13T07:32:03.141148+0000","last_active":"2024-04-13T07:32:39.023060+0000","last_peered":"2024-04-13T07:32:39.023060+0000","last_clean":"2024-04-13T07:32:39.023060+0000","last_became_active":"2024-04-13T07:32:03.139990+0000","last_became_peered":"2024-04-13T07:32:03.139990+0000","last_unstale":"2024-04-13T07:32:39.023060+0000","last_undegraded":"2024-04-13T07:32:39.023060+0000","last_fullsized":"2024-04-13T07:32:39.023060+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:18:49.325969+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022716+0000","last_change":"2024-04-13T07:32:03.077062+0000","last_active":"2024-04-13T07:32:39.022716+0000","last_peered":"2024-04-13T07:32:39.022716+0000","last_clean":"2024-04-13T07:32:39.022716+0000","last_became_active":"2024-04-13T07:32:03.076337+0000","last_became_peered":"2024-04-13T07:32:03.076337+0000","last_unstale":"2024-04-13T07:32:39.022716+0000","last_undegraded":"2024-04-13T07:32:39.022716+0000","last_fullsized":"2024-04-13T07:32:39.022716+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:22:01.092239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022844+0000","last_change":"2024-04-13T07:32:03.144389+0000","last_active":"2024-04-13T07:32:39.022844+0000","last_peered":"2024-04-13T07:32:39.022844+0000","last_clean":"2024-04-13T07:32:39.022844+0000","last_became_active":"2024-04-13T07:32:03.143541+0000","last_became_peered":"2024-04-13T07:32:03.143541+0000","last_unstale":"2024-04-13T07:32:39.022844+0000","last_undegraded":"2024-04-13T07:32:39.022844+0000","last_fullsized":"2024-04-13T07:32:39.022844+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:14:17.879743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021970+0000","last_change":"2024-04-13T07:32:03.066107+0000","last_active":"2024-04-13T07:32:39.021970+0000","last_peered":"2024-04-13T07:32:39.021970+0000","last_clean":"2024-04-13T07:32:39.021970+0000","last_became_active":"2024-04-13T07:32:03.065235+0000","last_became_peered":"2024-04-13T07:32:03.065235+0000","last_unstale":"2024-04-13T07:32:39.021970+0000","last_undegraded":"2024-04-13T07:32:39.021970+0000","last_fullsized":"2024-04-13T07:32:39.021970+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:46:26.415643+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.024063+0000","last_change":"2024-04-13T07:32:03.141938+0000","last_active":"2024-04-13T07:32:39.024063+0000","last_peered":"2024-04-13T07:32:39.024063+0000","last_clean":"2024-04-13T07:32:39.024063+0000","last_became_active":"2024-04-13T07:32:03.141631+0000","last_became_peered":"2024-04-13T07:32:03.141631+0000","last_unstale":"2024-04-13T07:32:39.024063+0000","last_undegraded":"2024-04-13T07:32:39.024063+0000","last_fullsized":"2024-04-13T07:32:39.024063+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:42:20.084568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010665+0000","last_change":"2024-04-13T07:32:03.083518+0000","last_active":"2024-04-13T07:32:39.010665+0000","last_peered":"2024-04-13T07:32:39.010665+0000","last_clean":"2024-04-13T07:32:39.010665+0000","last_became_active":"2024-04-13T07:32:03.082695+0000","last_became_peered":"2024-04-13T07:32:03.082695+0000","last_unstale":"2024-04-13T07:32:39.010665+0000","last_undegraded":"2024-04-13T07:32:39.010665+0000","last_fullsized":"2024-04-13T07:32:39.010665+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:58:25.148756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022948+0000","last_change":"2024-04-13T07:32:03.140777+0000","last_active":"2024-04-13T07:32:39.022948+0000","last_peered":"2024-04-13T07:32:39.022948+0000","last_clean":"2024-04-13T07:32:39.022948+0000","last_became_active":"2024-04-13T07:32:03.140026+0000","last_became_peered":"2024-04-13T07:32:03.140026+0000","last_unstale":"2024-04-13T07:32:39.022948+0000","last_undegraded":"2024-04-13T07:32:39.022948+0000","last_fullsized":"2024-04-13T07:32:39.022948+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:03:27.099045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":17,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.011729+0000","last_change":"2024-04-13T07:32:03.063483+0000","last_active":"2024-04-13T07:32:39.011729+0000","last_peered":"2024-04-13T07:32:39.011729+0000","last_clean":"2024-04-13T07:32:39.011729+0000","last_became_active":"2024-04-13T07:32:03.062230+0000","last_became_peered":"2024-04-13T07:32:03.062230+0000","last_unstale":"2024-04-13T07:32:39.011729+0000","last_undegraded":"2024-04-13T07:32:39.011729+0000","last_fullsized":"2024-04-13T07:32:39.011729+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:40:08.222569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148756+0000","last_change":"2024-04-13T07:30:21.470196+0000","last_active":"2024-04-13T07:32:39.148756+0000","last_peered":"2024-04-13T07:32:39.148756+0000","last_clean":"2024-04-13T07:32:39.148756+0000","last_became_active":"2024-04-13T07:30:21.469340+0000","last_became_peered":"2024-04-13T07:30:21.469340+0000","last_unstale":"2024-04-13T07:32:39.148756+0000","last_undegraded":"2024-04-13T07:32:39.148756+0000","last_fullsized":"2024-04-13T07:32:39.148756+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:32:48.529436+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.016477+0000","last_change":"2024-04-13T07:32:04.424228+0000","last_active":"2024-04-13T07:32:39.016477+0000","last_peered":"2024-04-13T07:32:39.016477+0000","last_clean":"2024-04-13T07:32:39.016477+0000","last_became_active":"2024-04-13T07:32:04.424027+0000","last_became_peered":"2024-04-13T07:32:04.424027+0000","last_unstale":"2024-04-13T07:32:39.016477+0000","last_undegraded":"2024-04-13T07:32:39.016477+0000","last_fullsized":"2024-04-13T07:32:39.016477+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:53:59.376447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.019359+0000","last_change":"2024-04-13T07:30:22.257370+0000","last_active":"2024-04-13T07:32:39.019359+0000","last_peered":"2024-04-13T07:32:39.019359+0000","last_clean":"2024-04-13T07:32:39.019359+0000","last_became_active":"2024-04-13T07:30:22.256904+0000","last_became_peered":"2024-04-13T07:30:22.256904+0000","last_unstale":"2024-04-13T07:32:39.019359+0000","last_undegraded":"2024-04-13T07:32:39.019359+0000","last_fullsized":"2024-04-13T07:32:39.019359+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:09:42.362257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.019750+0000","last_change":"2024-04-13T07:32:03.092985+0000","last_active":"2024-04-13T07:32:39.019750+0000","last_peered":"2024-04-13T07:32:39.019750+0000","last_clean":"2024-04-13T07:32:39.019750+0000","last_became_active":"2024-04-13T07:32:03.092332+0000","last_became_peered":"2024-04-13T07:32:03.092332+0000","last_unstale":"2024-04-13T07:32:39.019750+0000","last_undegraded":"2024-04-13T07:32:39.019750+0000","last_fullsized":"2024-04-13T07:32:39.019750+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:06:46.635980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010276+0000","last_change":"2024-04-13T07:32:04.425960+0000","last_active":"2024-04-13T07:32:39.010276+0000","last_peered":"2024-04-13T07:32:39.010276+0000","last_clean":"2024-04-13T07:32:39.010276+0000","last_became_active":"2024-04-13T07:32:04.425616+0000","last_became_peered":"2024-04-13T07:32:04.425616+0000","last_unstale":"2024-04-13T07:32:39.010276+0000","last_undegraded":"2024-04-13T07:32:39.010276+0000","last_fullsized":"2024-04-13T07:32:39.010276+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:10:37.088266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013622+0000","last_change":"2024-04-13T07:32:03.080050+0000","last_active":"2024-04-13T07:32:39.013622+0000","last_peered":"2024-04-13T07:32:39.013622+0000","last_clean":"2024-04-13T07:32:39.013622+0000","last_became_active":"2024-04-13T07:32:03.078878+0000","last_became_peered":"2024-04-13T07:32:03.078878+0000","last_unstale":"2024-04-13T07:32:39.013622+0000","last_undegraded":"2024-04-13T07:32:39.013622+0000","last_fullsized":"2024-04-13T07:32:39.013622+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:10:05.791661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010138+0000","last_change":"2024-04-13T07:30:22.258054+0000","last_active":"2024-04-13T07:32:39.010138+0000","last_peered":"2024-04-13T07:32:39.010138+0000","last_clean":"2024-04-13T07:32:39.010138+0000","last_became_active":"2024-04-13T07:30:22.256961+0000","last_became_peered":"2024-04-13T07:30:22.256961+0000","last_unstale":"2024-04-13T07:32:39.010138+0000","last_undegraded":"2024-04-13T07:32:39.010138+0000","last_fullsized":"2024-04-13T07:32:39.010138+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:13:36.989961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.012043+0000","last_change":"2024-04-13T07:32:04.418392+0000","last_active":"2024-04-13T07:32:39.012043+0000","last_peered":"2024-04-13T07:32:39.012043+0000","last_clean":"2024-04-13T07:32:39.012043+0000","last_became_active":"2024-04-13T07:32:04.418051+0000","last_became_peered":"2024-04-13T07:32:04.418051+0000","last_unstale":"2024-04-13T07:32:39.012043+0000","last_undegraded":"2024-04-13T07:32:39.012043+0000","last_fullsized":"2024-04-13T07:32:39.012043+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:43:31.028251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009957+0000","last_change":"2024-04-13T07:32:03.069788+0000","last_active":"2024-04-13T07:32:39.009957+0000","last_peered":"2024-04-13T07:32:39.009957+0000","last_clean":"2024-04-13T07:32:39.009957+0000","last_became_active":"2024-04-13T07:32:03.068886+0000","last_became_peered":"2024-04-13T07:32:03.068886+0000","last_unstale":"2024-04-13T07:32:39.009957+0000","last_undegraded":"2024-04-13T07:32:39.009957+0000","last_fullsized":"2024-04-13T07:32:39.009957+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:41:50.507675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009382+0000","last_change":"2024-04-13T07:30:21.472711+0000","last_active":"2024-04-13T07:32:39.009382+0000","last_peered":"2024-04-13T07:32:39.009382+0000","last_clean":"2024-04-13T07:32:39.009382+0000","last_became_active":"2024-04-13T07:30:21.471979+0000","last_became_peered":"2024-04-13T07:30:21.471979+0000","last_unstale":"2024-04-13T07:32:39.009382+0000","last_undegraded":"2024-04-13T07:32:39.009382+0000","last_fullsized":"2024-04-13T07:32:39.009382+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:27:07.929194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148712+0000","last_change":"2024-04-13T07:32:04.423737+0000","last_active":"2024-04-13T07:32:39.148712+0000","last_peered":"2024-04-13T07:32:39.148712+0000","last_clean":"2024-04-13T07:32:39.148712+0000","last_became_active":"2024-04-13T07:32:04.421863+0000","last_became_peered":"2024-04-13T07:32:04.421863+0000","last_unstale":"2024-04-13T07:32:39.148712+0000","last_undegraded":"2024-04-13T07:32:39.148712+0000","last_fullsized":"2024-04-13T07:32:39.148712+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:12:20.475957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017110+0000","last_change":"2024-04-13T07:32:03.072842+0000","last_active":"2024-04-13T07:32:39.017110+0000","last_peered":"2024-04-13T07:32:39.017110+0000","last_clean":"2024-04-13T07:32:39.017110+0000","last_became_active":"2024-04-13T07:32:03.071974+0000","last_became_peered":"2024-04-13T07:32:03.071974+0000","last_unstale":"2024-04-13T07:32:39.017110+0000","last_undegraded":"2024-04-13T07:32:39.017110+0000","last_fullsized":"2024-04-13T07:32:39.017110+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:56:10.031268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":269,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-04-13T07:32:40.238221+0000","last_change":"2024-04-13T07:28:43.834029+0000","last_active":"2024-04-13T07:32:40.238221+0000","last_peered":"2024-04-13T07:32:40.238221+0000","last_clean":"2024-04-13T07:32:40.238221+0000","last_became_active":"2024-04-13T07:28:43.833192+0000","last_became_peered":"2024-04-13T07:28:43.833192+0000","last_unstale":"2024-04-13T07:32:40.238221+0000","last_undegraded":"2024-04-13T07:32:40.238221+0000","last_fullsized":"2024-04-13T07:32:40.238221+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_clean_scrub_stamp":"2024-04-13T07:24:24.657602+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:32:17.256871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021717+0000","last_change":"2024-04-13T07:30:21.467433+0000","last_active":"2024-04-13T07:32:39.021717+0000","last_peered":"2024-04-13T07:32:39.021717+0000","last_clean":"2024-04-13T07:32:39.021717+0000","last_became_active":"2024-04-13T07:30:21.466602+0000","last_became_peered":"2024-04-13T07:30:21.466602+0000","last_unstale":"2024-04-13T07:32:39.021717+0000","last_undegraded":"2024-04-13T07:32:39.021717+0000","last_fullsized":"2024-04-13T07:32:39.021717+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:29:15.004301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.016883+0000","last_change":"2024-04-13T07:32:04.102888+0000","last_active":"2024-04-13T07:32:39.016883+0000","last_peered":"2024-04-13T07:32:39.016883+0000","last_clean":"2024-04-13T07:32:39.016883+0000","last_became_active":"2024-04-13T07:32:04.102086+0000","last_became_peered":"2024-04-13T07:32:04.102086+0000","last_unstale":"2024-04-13T07:32:39.016883+0000","last_undegraded":"2024-04-13T07:32:39.016883+0000","last_fullsized":"2024-04-13T07:32:39.016883+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:19:16.706609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010342+0000","last_change":"2024-04-13T07:32:03.077038+0000","last_active":"2024-04-13T07:32:39.010342+0000","last_peered":"2024-04-13T07:32:39.010342+0000","last_clean":"2024-04-13T07:32:39.010342+0000","last_became_active":"2024-04-13T07:32:03.076284+0000","last_became_peered":"2024-04-13T07:32:03.076284+0000","last_unstale":"2024-04-13T07:32:39.010342+0000","last_undegraded":"2024-04-13T07:32:39.010342+0000","last_fullsized":"2024-04-13T07:32:39.010342+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:01:47.340845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023330+0000","last_change":"2024-04-13T07:30:21.471551+0000","last_active":"2024-04-13T07:32:39.023330+0000","last_peered":"2024-04-13T07:32:39.023330+0000","last_clean":"2024-04-13T07:32:39.023330+0000","last_became_active":"2024-04-13T07:30:21.470542+0000","last_became_peered":"2024-04-13T07:30:21.470542+0000","last_unstale":"2024-04-13T07:32:39.023330+0000","last_undegraded":"2024-04-13T07:32:39.023330+0000","last_fullsized":"2024-04-13T07:32:39.023330+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:54:43.387491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013383+0000","last_change":"2024-04-13T07:32:04.418369+0000","last_active":"2024-04-13T07:32:39.013383+0000","last_peered":"2024-04-13T07:32:39.013383+0000","last_clean":"2024-04-13T07:32:39.013383+0000","last_became_active":"2024-04-13T07:32:04.418037+0000","last_became_peered":"2024-04-13T07:32:04.418037+0000","last_unstale":"2024-04-13T07:32:39.013383+0000","last_undegraded":"2024-04-13T07:32:39.013383+0000","last_fullsized":"2024-04-13T07:32:39.013383+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:54:59.849559+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022408+0000","last_change":"2024-04-13T07:32:03.071040+0000","last_active":"2024-04-13T07:32:39.022408+0000","last_peered":"2024-04-13T07:32:39.022408+0000","last_clean":"2024-04-13T07:32:39.022408+0000","last_became_active":"2024-04-13T07:32:03.069086+0000","last_became_peered":"2024-04-13T07:32:03.069086+0000","last_unstale":"2024-04-13T07:32:39.022408+0000","last_undegraded":"2024-04-13T07:32:39.022408+0000","last_fullsized":"2024-04-13T07:32:39.022408+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:12:41.759089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021951+0000","last_change":"2024-04-13T07:30:21.469386+0000","last_active":"2024-04-13T07:32:39.021951+0000","last_peered":"2024-04-13T07:32:39.021951+0000","last_clean":"2024-04-13T07:32:39.021951+0000","last_became_active":"2024-04-13T07:30:21.468314+0000","last_became_peered":"2024-04-13T07:30:21.468314+0000","last_unstale":"2024-04-13T07:32:39.021951+0000","last_undegraded":"2024-04-13T07:32:39.021951+0000","last_fullsized":"2024-04-13T07:32:39.021951+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:51:04.281850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.011514+0000","last_change":"2024-04-13T07:32:04.374209+0000","last_active":"2024-04-13T07:32:39.011514+0000","last_peered":"2024-04-13T07:32:39.011514+0000","last_clean":"2024-04-13T07:32:39.011514+0000","last_became_active":"2024-04-13T07:32:04.373393+0000","last_became_peered":"2024-04-13T07:32:04.373393+0000","last_unstale":"2024-04-13T07:32:39.011514+0000","last_undegraded":"2024-04-13T07:32:39.011514+0000","last_fullsized":"2024-04-13T07:32:39.011514+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:57:46.913705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022449+0000","last_change":"2024-04-13T07:32:03.076043+0000","last_active":"2024-04-13T07:32:39.022449+0000","last_peered":"2024-04-13T07:32:39.022449+0000","last_clean":"2024-04-13T07:32:39.022449+0000","last_became_active":"2024-04-13T07:32:03.075193+0000","last_became_peered":"2024-04-13T07:32:03.075193+0000","last_unstale":"2024-04-13T07:32:39.022449+0000","last_undegraded":"2024-04-13T07:32:39.022449+0000","last_fullsized":"2024-04-13T07:32:39.022449+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:07:57.007339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":21,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013164+0000","last_change":"2024-04-13T07:30:21.471173+0000","last_active":"2024-04-13T07:32:39.013164+0000","last_peered":"2024-04-13T07:32:39.013164+0000","last_clean":"2024-04-13T07:32:39.013164+0000","last_became_active":"2024-04-13T07:30:21.470193+0000","last_became_peered":"2024-04-13T07:30:21.470193+0000","last_unstale":"2024-04-13T07:32:39.013164+0000","last_undegraded":"2024-04-13T07:32:39.013164+0000","last_fullsized":"2024-04-13T07:32:39.013164+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:13:17.367790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":11,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-04-13T07:32:40.027446+0000","last_change":"2024-04-13T07:32:40.027446+0000","last_active":"2024-04-13T07:32:40.027446+0000","last_peered":"2024-04-13T07:32:40.027446+0000","last_clean":"2024-04-13T07:32:40.027446+0000","last_became_active":"2024-04-13T07:32:40.026430+0000","last_became_peered":"2024-04-13T07:32:40.026430+0000","last_unstale":"2024-04-13T07:32:40.027446+0000","last_undegraded":"2024-04-13T07:32:40.027446+0000","last_fullsized":"2024-04-13T07:32:40.027446+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:46:05.767985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.016554+0000","last_change":"2024-04-13T07:32:03.090544+0000","last_active":"2024-04-13T07:32:39.016554+0000","last_peered":"2024-04-13T07:32:39.016554+0000","last_clean":"2024-04-13T07:32:39.016554+0000","last_became_active":"2024-04-13T07:32:03.084852+0000","last_became_peered":"2024-04-13T07:32:03.084852+0000","last_unstale":"2024-04-13T07:32:39.016554+0000","last_undegraded":"2024-04-13T07:32:39.016554+0000","last_fullsized":"2024-04-13T07:32:39.016554+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:32:01.912602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.024037+0000","last_change":"2024-04-13T07:32:03.146367+0000","last_active":"2024-04-13T07:32:39.024037+0000","last_peered":"2024-04-13T07:32:39.024037+0000","last_clean":"2024-04-13T07:32:39.024037+0000","last_became_active":"2024-04-13T07:32:03.144578+0000","last_became_peered":"2024-04-13T07:32:03.144578+0000","last_unstale":"2024-04-13T07:32:39.024037+0000","last_undegraded":"2024-04-13T07:32:39.024037+0000","last_fullsized":"2024-04-13T07:32:39.024037+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:20:42.951481+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023234+0000","last_change":"2024-04-13T07:32:03.050715+0000","last_active":"2024-04-13T07:32:39.023234+0000","last_peered":"2024-04-13T07:32:39.023234+0000","last_clean":"2024-04-13T07:32:39.023234+0000","last_became_active":"2024-04-13T07:32:03.049589+0000","last_became_peered":"2024-04-13T07:32:03.049589+0000","last_unstale":"2024-04-13T07:32:39.023234+0000","last_undegraded":"2024-04-13T07:32:39.023234+0000","last_fullsized":"2024-04-13T07:32:39.023234+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:12:21.131281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.024522+0000","last_change":"2024-04-13T07:32:03.138119+0000","last_active":"2024-04-13T07:32:39.024522+0000","last_peered":"2024-04-13T07:32:39.024522+0000","last_clean":"2024-04-13T07:32:39.024522+0000","last_became_active":"2024-04-13T07:32:03.137391+0000","last_became_peered":"2024-04-13T07:32:03.137391+0000","last_unstale":"2024-04-13T07:32:39.024522+0000","last_undegraded":"2024-04-13T07:32:39.024522+0000","last_fullsized":"2024-04-13T07:32:39.024522+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:24:01.849939+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148632+0000","last_change":"2024-04-13T07:32:03.072704+0000","last_active":"2024-04-13T07:32:39.148632+0000","last_peered":"2024-04-13T07:32:39.148632+0000","last_clean":"2024-04-13T07:32:39.148632+0000","last_became_active":"2024-04-13T07:32:03.071880+0000","last_became_peered":"2024-04-13T07:32:03.071880+0000","last_unstale":"2024-04-13T07:32:39.148632+0000","last_undegraded":"2024-04-13T07:32:39.148632+0000","last_fullsized":"2024-04-13T07:32:39.148632+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:33:54.606613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023633+0000","last_change":"2024-04-13T07:32:03.146200+0000","last_active":"2024-04-13T07:32:39.023633+0000","last_peered":"2024-04-13T07:32:39.023633+0000","last_clean":"2024-04-13T07:32:39.023633+0000","last_became_active":"2024-04-13T07:32:03.145450+0000","last_became_peered":"2024-04-13T07:32:03.145450+0000","last_unstale":"2024-04-13T07:32:39.023633+0000","last_undegraded":"2024-04-13T07:32:39.023633+0000","last_fullsized":"2024-04-13T07:32:39.023633+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:03:02.507834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017533+0000","last_change":"2024-04-13T07:32:03.082757+0000","last_active":"2024-04-13T07:32:39.017533+0000","last_peered":"2024-04-13T07:32:39.017533+0000","last_clean":"2024-04-13T07:32:39.017533+0000","last_became_active":"2024-04-13T07:32:03.081868+0000","last_became_peered":"2024-04-13T07:32:03.081868+0000","last_unstale":"2024-04-13T07:32:39.017533+0000","last_undegraded":"2024-04-13T07:32:39.017533+0000","last_fullsized":"2024-04-13T07:32:39.017533+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:51:59.170277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2,"num_read_kb":2,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010075+0000","last_change":"2024-04-13T07:32:04.376133+0000","last_active":"2024-04-13T07:32:39.010075+0000","last_peered":"2024-04-13T07:32:39.010075+0000","last_clean":"2024-04-13T07:32:39.010075+0000","last_became_active":"2024-04-13T07:32:04.375185+0000","last_became_peered":"2024-04-13T07:32:04.375185+0000","last_unstale":"2024-04-13T07:32:39.010075+0000","last_undegraded":"2024-04-13T07:32:39.010075+0000","last_fullsized":"2024-04-13T07:32:39.010075+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:31:10.129701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021757+0000","last_change":"2024-04-13T07:32:03.067875+0000","last_active":"2024-04-13T07:32:39.021757+0000","last_peered":"2024-04-13T07:32:39.021757+0000","last_clean":"2024-04-13T07:32:39.021757+0000","last_became_active":"2024-04-13T07:32:03.067001+0000","last_became_peered":"2024-04-13T07:32:03.067001+0000","last_unstale":"2024-04-13T07:32:39.021757+0000","last_undegraded":"2024-04-13T07:32:39.021757+0000","last_fullsized":"2024-04-13T07:32:39.021757+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:37:50.584428+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148280+0000","last_change":"2024-04-13T07:32:04.423046+0000","last_active":"2024-04-13T07:32:39.148280+0000","last_peered":"2024-04-13T07:32:39.148280+0000","last_clean":"2024-04-13T07:32:39.148280+0000","last_became_active":"2024-04-13T07:32:04.421900+0000","last_became_peered":"2024-04-13T07:32:04.421900+0000","last_unstale":"2024-04-13T07:32:39.148280+0000","last_undegraded":"2024-04-13T07:32:39.148280+0000","last_fullsized":"2024-04-13T07:32:39.148280+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:21:04.585612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010306+0000","last_change":"2024-04-13T07:32:03.078696+0000","last_active":"2024-04-13T07:32:39.010306+0000","last_peered":"2024-04-13T07:32:39.010306+0000","last_clean":"2024-04-13T07:32:39.010306+0000","last_became_active":"2024-04-13T07:32:03.077798+0000","last_became_peered":"2024-04-13T07:32:03.077798+0000","last_unstale":"2024-04-13T07:32:39.010306+0000","last_undegraded":"2024-04-13T07:32:39.010306+0000","last_fullsized":"2024-04-13T07:32:39.010306+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:32:02.676056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022535+0000","last_change":"2024-04-13T07:32:03.143672+0000","last_active":"2024-04-13T07:32:39.022535+0000","last_peered":"2024-04-13T07:32:39.022535+0000","last_clean":"2024-04-13T07:32:39.022535+0000","last_became_active":"2024-04-13T07:32:03.143068+0000","last_became_peered":"2024-04-13T07:32:03.143068+0000","last_unstale":"2024-04-13T07:32:39.022535+0000","last_undegraded":"2024-04-13T07:32:39.022535+0000","last_fullsized":"2024-04-13T07:32:39.022535+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:47:11.391489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010283+0000","last_change":"2024-04-13T07:32:03.079072+0000","last_active":"2024-04-13T07:32:39.010283+0000","last_peered":"2024-04-13T07:32:39.010283+0000","last_clean":"2024-04-13T07:32:39.010283+0000","last_became_active":"2024-04-13T07:32:03.078769+0000","last_became_peered":"2024-04-13T07:32:03.078769+0000","last_unstale":"2024-04-13T07:32:39.010283+0000","last_undegraded":"2024-04-13T07:32:39.010283+0000","last_fullsized":"2024-04-13T07:32:39.010283+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:25:42.191631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021493+0000","last_change":"2024-04-13T07:32:03.141831+0000","last_active":"2024-04-13T07:32:39.021493+0000","last_peered":"2024-04-13T07:32:39.021493+0000","last_clean":"2024-04-13T07:32:39.021493+0000","last_became_active":"2024-04-13T07:32:03.141053+0000","last_became_peered":"2024-04-13T07:32:03.141053+0000","last_unstale":"2024-04-13T07:32:39.021493+0000","last_undegraded":"2024-04-13T07:32:39.021493+0000","last_fullsized":"2024-04-13T07:32:39.021493+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:24:08.376151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010782+0000","last_change":"2024-04-13T07:32:03.093924+0000","last_active":"2024-04-13T07:32:39.010782+0000","last_peered":"2024-04-13T07:32:39.010782+0000","last_clean":"2024-04-13T07:32:39.010782+0000","last_became_active":"2024-04-13T07:32:03.093273+0000","last_became_peered":"2024-04-13T07:32:03.093273+0000","last_unstale":"2024-04-13T07:32:39.010782+0000","last_undegraded":"2024-04-13T07:32:39.010782+0000","last_fullsized":"2024-04-13T07:32:39.010782+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:53:00.895622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-04-13T07:32:40.240677+0000","last_change":"2024-04-13T07:32:40.240677+0000","last_active":"2024-04-13T07:32:40.240677+0000","last_peered":"2024-04-13T07:32:40.240677+0000","last_clean":"2024-04-13T07:32:40.240677+0000","last_became_active":"2024-04-13T07:32:40.240094+0000","last_became_peered":"2024-04-13T07:32:40.240094+0000","last_unstale":"2024-04-13T07:32:40.240677+0000","last_undegraded":"2024-04-13T07:32:40.240677+0000","last_fullsized":"2024-04-13T07:32:40.240677+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:49:33.573507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":32,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148825+0000","last_change":"2024-04-13T07:32:03.064053+0000","last_active":"2024-04-13T07:32:39.148825+0000","last_peered":"2024-04-13T07:32:39.148825+0000","last_clean":"2024-04-13T07:32:39.148825+0000","last_became_active":"2024-04-13T07:32:03.061879+0000","last_became_peered":"2024-04-13T07:32:03.061879+0000","last_unstale":"2024-04-13T07:32:39.148825+0000","last_undegraded":"2024-04-13T07:32:39.148825+0000","last_fullsized":"2024-04-13T07:32:39.148825+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:03:11.892760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009637+0000","last_change":"2024-04-13T07:32:04.423037+0000","last_active":"2024-04-13T07:32:39.009637+0000","last_peered":"2024-04-13T07:32:39.009637+0000","last_clean":"2024-04-13T07:32:39.009637+0000","last_became_active":"2024-04-13T07:32:04.422667+0000","last_became_peered":"2024-04-13T07:32:04.422667+0000","last_unstale":"2024-04-13T07:32:39.009637+0000","last_undegraded":"2024-04-13T07:32:39.009637+0000","last_fullsized":"2024-04-13T07:32:39.009637+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:07:35.236894+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009276+0000","last_change":"2024-04-13T07:32:03.082422+0000","last_active":"2024-04-13T07:32:39.009276+0000","last_peered":"2024-04-13T07:32:39.009276+0000","last_clean":"2024-04-13T07:32:39.009276+0000","last_became_active":"2024-04-13T07:32:03.080059+0000","last_became_peered":"2024-04-13T07:32:03.080059+0000","last_unstale":"2024-04-13T07:32:39.009276+0000","last_undegraded":"2024-04-13T07:32:39.009276+0000","last_fullsized":"2024-04-13T07:32:39.009276+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:07:47.268352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022163+0000","last_change":"2024-04-13T07:32:03.143977+0000","last_active":"2024-04-13T07:32:39.022163+0000","last_peered":"2024-04-13T07:32:39.022163+0000","last_clean":"2024-04-13T07:32:39.022163+0000","last_became_active":"2024-04-13T07:32:03.142886+0000","last_became_peered":"2024-04-13T07:32:03.142886+0000","last_unstale":"2024-04-13T07:32:39.022163+0000","last_undegraded":"2024-04-13T07:32:39.022163+0000","last_fullsized":"2024-04-13T07:32:39.022163+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:21:20.364627+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009638+0000","last_change":"2024-04-13T07:32:03.084887+0000","last_active":"2024-04-13T07:32:39.009638+0000","last_peered":"2024-04-13T07:32:39.009638+0000","last_clean":"2024-04-13T07:32:39.009638+0000","last_became_active":"2024-04-13T07:32:03.069100+0000","last_became_peered":"2024-04-13T07:32:03.069100+0000","last_unstale":"2024-04-13T07:32:39.009638+0000","last_undegraded":"2024-04-13T07:32:39.009638+0000","last_fullsized":"2024-04-13T07:32:39.009638+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:10:11.856147+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017427+0000","last_change":"2024-04-13T07:32:04.422844+0000","last_active":"2024-04-13T07:32:39.017427+0000","last_peered":"2024-04-13T07:32:39.017427+0000","last_clean":"2024-04-13T07:32:39.017427+0000","last_became_active":"2024-04-13T07:32:04.422538+0000","last_became_peered":"2024-04-13T07:32:04.422538+0000","last_unstale":"2024-04-13T07:32:39.017427+0000","last_undegraded":"2024-04-13T07:32:39.017427+0000","last_fullsized":"2024-04-13T07:32:39.017427+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:33:44.840955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":20,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009835+0000","last_change":"2024-04-13T07:32:03.081497+0000","last_active":"2024-04-13T07:32:39.009835+0000","last_peered":"2024-04-13T07:32:39.009835+0000","last_clean":"2024-04-13T07:32:39.009835+0000","last_became_active":"2024-04-13T07:32:03.080871+0000","last_became_peered":"2024-04-13T07:32:03.080871+0000","last_unstale":"2024-04-13T07:32:39.009835+0000","last_undegraded":"2024-04-13T07:32:39.009835+0000","last_fullsized":"2024-04-13T07:32:39.009835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:52:03.244799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022634+0000","last_change":"2024-04-13T07:32:03.144209+0000","last_active":"2024-04-13T07:32:39.022634+0000","last_peered":"2024-04-13T07:32:39.022634+0000","last_clean":"2024-04-13T07:32:39.022634+0000","last_became_active":"2024-04-13T07:32:03.141741+0000","last_became_peered":"2024-04-13T07:32:03.141741+0000","last_unstale":"2024-04-13T07:32:39.022634+0000","last_undegraded":"2024-04-13T07:32:39.022634+0000","last_fullsized":"2024-04-13T07:32:39.022634+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:36:52.296757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"70'7","reported_seq":29,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.023835+0000","last_change":"2024-04-13T07:32:03.074002+0000","last_active":"2024-04-13T07:32:39.023835+0000","last_peered":"2024-04-13T07:32:39.023835+0000","last_clean":"2024-04-13T07:32:39.023835+0000","last_became_active":"2024-04-13T07:32:03.073163+0000","last_became_peered":"2024-04-13T07:32:03.073163+0000","last_unstale":"2024-04-13T07:32:39.023835+0000","last_undegraded":"2024-04-13T07:32:39.023835+0000","last_fullsized":"2024-04-13T07:32:39.023835+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":7,"log_dups_size":0,"ondisk_log_size":7,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:50:04.054552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1564,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.011309+0000","last_change":"2024-04-13T07:32:04.418476+0000","last_active":"2024-04-13T07:32:39.011309+0000","last_peered":"2024-04-13T07:32:39.011309+0000","last_clean":"2024-04-13T07:32:39.011309+0000","last_became_active":"2024-04-13T07:32:04.418126+0000","last_became_peered":"2024-04-13T07:32:04.418126+0000","last_unstale":"2024-04-13T07:32:39.011309+0000","last_undegraded":"2024-04-13T07:32:39.011309+0000","last_fullsized":"2024-04-13T07:32:39.011309+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:24:25.424210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":22,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.009904+0000","last_change":"2024-04-13T07:32:03.087811+0000","last_active":"2024-04-13T07:32:39.009904+0000","last_peered":"2024-04-13T07:32:39.009904+0000","last_clean":"2024-04-13T07:32:39.009904+0000","last_became_active":"2024-04-13T07:32:03.087293+0000","last_became_peered":"2024-04-13T07:32:03.087293+0000","last_unstale":"2024-04-13T07:32:39.009904+0000","last_undegraded":"2024-04-13T07:32:39.009904+0000","last_fullsized":"2024-04-13T07:32:39.009904+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:07:16.065771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017771+0000","last_change":"2024-04-13T07:32:04.423028+0000","last_active":"2024-04-13T07:32:39.017771+0000","last_peered":"2024-04-13T07:32:39.017771+0000","last_clean":"2024-04-13T07:32:39.017771+0000","last_became_active":"2024-04-13T07:32:04.422663+0000","last_became_peered":"2024-04-13T07:32:04.422663+0000","last_unstale":"2024-04-13T07:32:39.017771+0000","last_undegraded":"2024-04-13T07:32:39.017771+0000","last_fullsized":"2024-04-13T07:32:39.017771+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:37:23.228475+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.021870+0000","last_change":"2024-04-13T07:32:03.068639+0000","last_active":"2024-04-13T07:32:39.021870+0000","last_peered":"2024-04-13T07:32:39.021870+0000","last_clean":"2024-04-13T07:32:39.021870+0000","last_became_active":"2024-04-13T07:32:03.067401+0000","last_became_peered":"2024-04-13T07:32:03.067401+0000","last_unstale":"2024-04-13T07:32:39.021870+0000","last_undegraded":"2024-04-13T07:32:39.021870+0000","last_fullsized":"2024-04-13T07:32:39.021870+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:51:09.165741+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.016703+0000","last_change":"2024-04-13T07:32:04.104373+0000","last_active":"2024-04-13T07:32:39.016703+0000","last_peered":"2024-04-13T07:32:39.016703+0000","last_clean":"2024-04-13T07:32:39.016703+0000","last_became_active":"2024-04-13T07:32:04.103428+0000","last_became_peered":"2024-04-13T07:32:04.103428+0000","last_unstale":"2024-04-13T07:32:39.016703+0000","last_undegraded":"2024-04-13T07:32:39.016703+0000","last_fullsized":"2024-04-13T07:32:39.016703+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:39:29.039199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.013235+0000","last_change":"2024-04-13T07:32:03.089122+0000","last_active":"2024-04-13T07:32:39.013235+0000","last_peered":"2024-04-13T07:32:39.013235+0000","last_clean":"2024-04-13T07:32:39.013235+0000","last_became_active":"2024-04-13T07:32:03.087087+0000","last_became_peered":"2024-04-13T07:32:03.087087+0000","last_unstale":"2024-04-13T07:32:39.013235+0000","last_undegraded":"2024-04-13T07:32:39.013235+0000","last_fullsized":"2024-04-13T07:32:39.013235+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:15:12.555631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.148826+0000","last_change":"2024-04-13T07:32:04.102324+0000","last_active":"2024-04-13T07:32:39.148826+0000","last_peered":"2024-04-13T07:32:39.148826+0000","last_clean":"2024-04-13T07:32:39.148826+0000","last_became_active":"2024-04-13T07:32:04.101488+0000","last_became_peered":"2024-04-13T07:32:04.101488+0000","last_unstale":"2024-04-13T07:32:39.148826+0000","last_undegraded":"2024-04-13T07:32:39.148826+0000","last_fullsized":"2024-04-13T07:32:39.148826+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:32:06.780234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.017007+0000","last_change":"2024-04-13T07:32:03.084186+0000","last_active":"2024-04-13T07:32:39.017007+0000","last_peered":"2024-04-13T07:32:39.017007+0000","last_clean":"2024-04-13T07:32:39.017007+0000","last_became_active":"2024-04-13T07:32:03.080930+0000","last_became_peered":"2024-04-13T07:32:03.080930+0000","last_unstale":"2024-04-13T07:32:39.017007+0000","last_undegraded":"2024-04-13T07:32:39.017007+0000","last_fullsized":"2024-04-13T07:32:39.017007+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:22:54.864468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":15,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.011288+0000","last_change":"2024-04-13T07:32:04.423862+0000","last_active":"2024-04-13T07:32:39.011288+0000","last_peered":"2024-04-13T07:32:39.011288+0000","last_clean":"2024-04-13T07:32:39.011288+0000","last_became_active":"2024-04-13T07:32:04.423514+0000","last_became_peered":"2024-04-13T07:32:04.423514+0000","last_unstale":"2024-04-13T07:32:39.011288+0000","last_undegraded":"2024-04-13T07:32:39.011288+0000","last_fullsized":"2024-04-13T07:32:39.011288+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":68,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:55:46.044341+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":16,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.010738+0000","last_change":"2024-04-13T07:32:03.081098+0000","last_active":"2024-04-13T07:32:39.010738+0000","last_peered":"2024-04-13T07:32:39.010738+0000","last_clean":"2024-04-13T07:32:39.010738+0000","last_became_active":"2024-04-13T07:32:03.080593+0000","last_became_peered":"2024-04-13T07:32:03.080593+0000","last_unstale":"2024-04-13T07:32:39.010738+0000","last_undegraded":"2024-04-13T07:32:39.010738+0000","last_fullsized":"2024-04-13T07:32:39.010738+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:07:33.886771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2024-04-13T07:32:40.032333+0000","last_change":"2024-04-13T07:32:40.032333+0000","last_active":"2024-04-13T07:32:40.032333+0000","last_peered":"2024-04-13T07:32:40.032333+0000","last_clean":"2024-04-13T07:32:40.032333+0000","last_became_active":"2024-04-13T07:32:40.031380+0000","last_became_peered":"2024-04-13T07:32:40.031380+0000","last_unstale":"2024-04-13T07:32:40.032333+0000","last_undegraded":"2024-04-13T07:32:40.032333+0000","last_fullsized":"2024-04-13T07:32:40.032333+0000","mapping_epoch":71,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":72,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:00:32.658370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":18,"reported_epoch":71,"state":"active+clean","last_fresh":"2024-04-13T07:32:39.022818+0000","last_change":"2024-04-13T07:32:03.060936+0000","last_active":"2024-04-13T07:32:39.022818+0000","last_peered":"2024-04-13T07:32:39.022818+0000","last_clean":"2024-04-13T07:32:39.022818+0000","last_became_active":"2024-04-13T07:32:03.059737+0000","last_became_peered":"2024-04-13T07:32:03.059737+0000","last_unstale":"2024-04-13T07:32:39.022818+0000","last_undegraded":"2024-04-13T07:32:39.022818+0000","last_fullsized":"2024-04-13T07:32:39.022818+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:44:10.168724+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2362,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":77,"num_read_kb":80,"num_write":45,"num_write_kb":14,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":7086,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201341,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32452,"kb_used_data":1332,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712700,"statfs":{"total":95995035648,"available":95961804800,"internally_reserved":0,"allocated":1363968,"data_stored":916725,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":31848907},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.88600000000000001,"15min":0.878},"min":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"max":{"1min":1.264,"5min":1.3,"15min":1.3},"last":0.66500000000000004},{"interface":"front","average":{"1min":0.91800000000000004,"5min":0.90700000000000003,"15min":0.92700000000000005},"min":{"1min":0.59599999999999997,"5min":0.51000000000000001,"15min":0.51000000000000001},"max":{"1min":1.548,"5min":1.548,"15min":1.548},"last":1.0189999999999999}]},{"osd":1,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":0.89400000000000002,"5min":0.88500000000000001,"15min":0.88},"min":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.3149999999999999,"5min":1.3220000000000001,"15min":1.3220000000000001},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.96199999999999997,"15min":0.95999999999999996},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":1.391,"5min":1.391,"15min":1.391},"last":0.76600000000000001}]},{"osd":2,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":0.95799999999999996,"5min":0.92600000000000005,"15min":0.92300000000000004},"min":{"1min":0.53300000000000003,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.397,"5min":1.397,"15min":1.397},"last":0.71199999999999997},{"interface":"front","average":{"1min":0.92200000000000004,"5min":0.999,"15min":1.0129999999999999},"min":{"1min":0.56899999999999995,"5min":0.53300000000000003,"15min":0.53300000000000003},"max":{"1min":1.2549999999999999,"5min":1.369,"15min":1.369},"last":1.274}]},{"osd":3,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":1.0009999999999999,"5min":0.94899999999999995,"15min":0.94499999999999995},"min":{"1min":0.55600000000000005,"5min":0.52000000000000002,"15min":0.52000000000000002},"max":{"1min":1.304,"5min":1.401,"15min":1.401},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.0720000000000001,"5min":1.0529999999999999,"15min":1.0449999999999999},"min":{"1min":0.70199999999999996,"5min":0.501,"15min":0.501},"max":{"1min":1.3460000000000001,"5min":1.4239999999999999,"15min":1.4239999999999999},"last":1.4179999999999999}]},{"osd":4,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.91200000000000003,"15min":0.88500000000000001},"min":{"1min":0.45600000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.3420000000000001,"5min":1.3420000000000001,"15min":1.3420000000000001},"last":1.133},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.95199999999999996,"15min":0.93500000000000005},"min":{"1min":0.502,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":1.506,"5min":1.506,"15min":1.506},"last":1.1559999999999999}]},{"osd":5,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":1.1319999999999999,"5min":1.028,"15min":1.006},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":1.407,"5min":1.492,"15min":1.492},"last":1.48},{"interface":"front","average":{"1min":1.147,"5min":1.0669999999999999,"15min":1.0529999999999999},"min":{"1min":0.59599999999999997,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":1.474,"5min":1.548,"15min":1.548},"last":1.3220000000000001}]},{"osd":6,"last update":"Sat Apr 13 07:32:46 2024","interfaces":[{"interface":"back","average":{"1min":1.125,"5min":1.109,"15min":1.1020000000000001},"min":{"1min":0.64100000000000001,"5min":0.52800000000000002,"15min":0.52800000000000002},"max":{"1min":1.337,"5min":1.536,"15min":1.536},"last":1.4490000000000001},{"interface":"front","average":{"1min":1.1180000000000001,"5min":1.0900000000000001,"15min":1.0980000000000001},"min":{"1min":0.68899999999999995,"5min":0.434,"15min":0.434},"max":{"1min":1.4590000000000001,"5min":1.633,"15min":1.633},"last":1.3660000000000001}]}]},{"osd":6,"up_from":47,"seq":201863462983,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28332,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93716820,"statfs":{"total":95995035648,"available":95966023680,"internally_reserved":0,"allocated":1343488,"data_stored":915975,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.91300000000000003,"5min":0.90000000000000002,"15min":0.85899999999999999},"min":{"1min":0.56899999999999995,"5min":0.53600000000000003,"15min":0.53600000000000003},"max":{"1min":1.145,"5min":1.5109999999999999,"15min":1.5109999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":0.995,"5min":0.93400000000000005,"15min":0.89600000000000002},"min":{"1min":0.67000000000000004,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.2310000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"last":1.194}]},{"osd":1,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.92800000000000005,"15min":0.92400000000000004},"min":{"1min":0.60999999999999999,"5min":0.53000000000000003,"15min":0.53000000000000003},"max":{"1min":1.444,"5min":1.444,"15min":1.444},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.90200000000000002,"5min":0.92200000000000004,"15min":0.93000000000000005},"min":{"1min":0.58199999999999996,"5min":0.58199999999999996,"15min":0.58199999999999996},"max":{"1min":1.167,"5min":1.4710000000000001,"15min":1.4710000000000001},"last":0.78600000000000003}]},{"osd":2,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.97999999999999998,"5min":0.95499999999999996,"15min":0.91300000000000003},"min":{"1min":0.57099999999999995,"5min":0.55000000000000004,"15min":0.55000000000000004},"max":{"1min":1.2270000000000001,"5min":1.2849999999999999,"15min":1.2849999999999999},"last":0.85799999999999998},{"interface":"front","average":{"1min":0.96199999999999997,"5min":0.97899999999999998,"15min":1.004},"min":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"max":{"1min":1.2969999999999999,"5min":1.2969999999999999,"15min":1.2969999999999999},"last":1.0049999999999999}]},{"osd":3,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.97799999999999998,"5min":0.95399999999999996,"15min":0.93000000000000005},"min":{"1min":0.58799999999999997,"5min":0.52500000000000002,"15min":0.52500000000000002},"max":{"1min":1.3169999999999999,"5min":1.45,"15min":1.45},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.046,"5min":1.0209999999999999,"15min":1.0209999999999999},"min":{"1min":0.70299999999999996,"5min":0.61899999999999999,"15min":0.61899999999999999},"max":{"1min":1.302,"5min":1.5660000000000001,"15min":1.5660000000000001},"last":1.123}]},{"osd":4,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":0.92000000000000004,"15min":0.90200000000000002},"min":{"1min":0.745,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":1.3340000000000001,"5min":1.3340000000000001,"15min":1.3340000000000001},"last":1.2509999999999999},{"interface":"front","average":{"1min":0.99399999999999999,"5min":0.94399999999999995,"15min":0.87},"min":{"1min":0.68300000000000005,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.333,"5min":1.5,"15min":1.5},"last":1.0800000000000001}]},{"osd":5,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":1.0449999999999999,"5min":0.99199999999999999,"15min":0.92600000000000005},"min":{"1min":0.749,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":1.286,"5min":1.423,"15min":1.423},"last":1.3420000000000001},{"interface":"front","average":{"1min":1.117,"5min":1.038,"15min":0.997},"min":{"1min":0.78200000000000003,"5min":0.38,"15min":0.38},"max":{"1min":1.349,"5min":1.7470000000000001,"15min":1.7470000000000001},"last":1.1659999999999999}]},{"osd":7,"last update":"Sat Apr 13 07:32:43 2024","interfaces":[{"interface":"back","average":{"1min":1.179,"5min":1.135,"15min":1.147},"min":{"1min":0.80100000000000005,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":1.5149999999999999,"5min":1.6930000000000001,"15min":1.6930000000000001},"last":1.2290000000000001},{"interface":"front","average":{"1min":1.169,"5min":1.1299999999999999,"15min":1.1419999999999999},"min":{"1min":0.79600000000000004,"5min":0.44,"15min":0.44},"max":{"1min":1.599,"5min":1.599,"15min":1.599},"last":1.294}]}]},{"osd":1,"up_from":13,"seq":55834574970,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":724,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":741376,"data_stored":324577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:32:51 2024","interfaces":[{"interface":"back","average":{"1min":0.67200000000000004,"5min":0.65900000000000003,"15min":0.57699999999999996},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":1.0980000000000001,"5min":1.1699999999999999,"15min":1.1699999999999999},"last":1.02},{"interface":"front","average":{"1min":0.84099999999999997,"5min":0.85099999999999998,"15min":0.68899999999999995},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":1.3839999999999999,"5min":1.3839999999999999,"15min":1.3839999999999999},"last":1.155}]},{"osd":2,"last update":"Sat Apr 13 07:32:32 2024","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.70499999999999996,"15min":0.65600000000000003},"min":{"1min":0.38600000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.3200000000000001,"5min":1.3200000000000001,"15min":1.3200000000000001},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.80700000000000005,"5min":0.84499999999999997,"15min":0.74099999999999999},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.27900000000000003},"max":{"1min":1.26,"5min":1.3009999999999999,"15min":1.3009999999999999},"last":0.93200000000000005}]},{"osd":3,"last update":"Sat Apr 13 07:33:24 2024","interfaces":[{"interface":"back","average":{"1min":0.80000000000000004,"5min":0.86299999999999999,"15min":0.79200000000000004},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":1.22,"5min":1.4350000000000001,"15min":1.4350000000000001},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.95599999999999996,"5min":0.996,"15min":0.90500000000000003},"min":{"1min":0.39500000000000002,"5min":0.33200000000000002,"15min":0.33200000000000002},"max":{"1min":1.357,"5min":1.7030000000000001,"15min":1.7030000000000001},"last":1.0609999999999999}]},{"osd":4,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91800000000000004,"15min":0.91700000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":1.302,"5min":1.349,"15min":1.349},"last":0.81899999999999995},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.0089999999999999,"15min":0.95299999999999996},"min":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"max":{"1min":1.3500000000000001,"5min":1.998,"15min":1.998},"last":0.85599999999999998}]},{"osd":5,"last update":"Sat Apr 13 07:33:07 2024","interfaces":[{"interface":"back","average":{"1min":0.98299999999999998,"5min":1.0640000000000001,"15min":1.044},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":1.4530000000000001,"5min":2.0449999999999999,"15min":2.0449999999999999},"last":1.25},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.004,"15min":0.99199999999999999},"min":{"1min":0.61799999999999999,"5min":0.59899999999999998,"15min":0.59899999999999998},"max":{"1min":1.413,"5min":1.413,"15min":1.413},"last":0.88400000000000001}]},{"osd":6,"last update":"Sat Apr 13 07:32:56 2024","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.0349999999999999,"15min":1.026},"min":{"1min":0.59999999999999998,"5min":0.59999999999999998,"15min":0.59999999999999998},"max":{"1min":1.6419999999999999,"5min":1.6419999999999999,"15min":1.6419999999999999},"last":0.98399999999999999},{"interface":"front","average":{"1min":1.2050000000000001,"5min":1.157,"15min":1.1579999999999999},"min":{"1min":0.71199999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"max":{"1min":1.954,"5min":1.954,"15min":1.954},"last":1.2090000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:32:51 2024","interfaces":[{"interface":"back","average":{"1min":1.1299999999999999,"5min":1.1259999999999999,"15min":1.1439999999999999},"min":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"max":{"1min":1.532,"5min":1.532,"15min":1.532},"last":1.123},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.1459999999999999,"15min":1.1759999999999999},"min":{"1min":0.622,"5min":0.622,"15min":0.622},"max":{"1min":2.214,"5min":2.214,"15min":2.214},"last":1.0880000000000001}]}]},{"osd":0,"up_from":8,"seq":34359738501,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32432,"kb_used_data":1312,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93712720,"statfs":{"total":95995035648,"available":95961825280,"internally_reserved":0,"allocated":1343488,"data_stored":915549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Apr 13 07:32:50 2024","interfaces":[{"interface":"back","average":{"1min":0.67300000000000004,"5min":0.69199999999999995,"15min":0.58899999999999997},"min":{"1min":0.32400000000000001,"5min":0.29699999999999999,"15min":0.26500000000000001},"max":{"1min":1.036,"5min":1.1140000000000001,"15min":1.1910000000000001},"last":0.88},{"interface":"front","average":{"1min":0.79600000000000004,"5min":0.78800000000000003,"15min":0.64700000000000002},"min":{"1min":0.5,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.0109999999999999,"5min":1.3160000000000001,"15min":1.3160000000000001},"last":0.63700000000000001}]},{"osd":2,"last update":"Sat Apr 13 07:32:30 2024","interfaces":[{"interface":"back","average":{"1min":0.76100000000000001,"5min":0.76200000000000001,"15min":0.69899999999999995},"min":{"1min":0.39800000000000002,"5min":0.311,"15min":0.311},"max":{"1min":1.0960000000000001,"5min":1.1359999999999999,"15min":1.1359999999999999},"last":0.91800000000000004},{"interface":"front","average":{"1min":0.75600000000000001,"5min":0.78100000000000003,"15min":0.70399999999999996},"min":{"1min":0.442,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":1.0800000000000001,"5min":1.1919999999999999,"15min":1.2},"last":0.73199999999999998}]},{"osd":3,"last update":"Sat Apr 13 07:33:29 2024","interfaces":[{"interface":"back","average":{"1min":0.72499999999999998,"5min":0.77600000000000002,"15min":0.76000000000000001},"min":{"1min":0.39400000000000002,"5min":0.36099999999999999,"15min":0.36099999999999999},"max":{"1min":0.95699999999999996,"5min":1.3979999999999999,"15min":1.3979999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.77900000000000003,"5min":0.82999999999999996,"15min":0.81899999999999995},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.2430000000000001,"5min":1.2430000000000001,"15min":1.2629999999999999},"last":0.83799999999999997}]},{"osd":4,"last update":"Sat Apr 13 07:33:11 2024","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.99299999999999999,"15min":0.95399999999999996},"min":{"1min":0.59999999999999998,"5min":0.57599999999999996,"15min":0.57599999999999996},"max":{"1min":1.2110000000000001,"5min":1.3859999999999999,"15min":1.3859999999999999},"last":0.96099999999999997},{"interface":"front","average":{"1min":1.0409999999999999,"5min":1.0600000000000001,"15min":1.014},"min":{"1min":0.59199999999999997,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.3320000000000001,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":1.0209999999999999}]},{"osd":5,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.995,"5min":1.0720000000000001,"15min":1.024},"min":{"1min":0.60199999999999998,"5min":0.501,"15min":0.501},"max":{"1min":1.55,"5min":1.748,"15min":1.748},"last":1.161},{"interface":"front","average":{"1min":1.044,"5min":1.0600000000000001,"15min":1.008},"min":{"1min":0.626,"5min":0.56599999999999995,"15min":0.56599999999999995},"max":{"1min":1.6339999999999999,"5min":1.6339999999999999,"15min":1.6339999999999999},"last":0.999}]},{"osd":6,"last update":"Sat Apr 13 07:32:57 2024","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.135,"15min":1.1659999999999999},"min":{"1min":0.64000000000000001,"5min":0.58499999999999996,"15min":0.58499999999999996},"max":{"1min":1.5569999999999999,"5min":1.621,"15min":1.621},"last":1.111},{"interface":"front","average":{"1min":1.054,"5min":1.1100000000000001,"15min":1.0960000000000001},"min":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"max":{"1min":1.494,"5min":1.659,"15min":1.659},"last":1.0580000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:32:50 2024","interfaces":[{"interface":"back","average":{"1min":1.1819999999999999,"5min":1.181,"15min":1.218},"min":{"1min":0.67600000000000005,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.879,"5min":1.879,"15min":1.879},"last":1.0980000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.23,"15min":1.24},"min":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"max":{"1min":1.7909999999999999,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.2090000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411440,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":732,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":749568,"data_stored":325193,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.71399999999999997,"5min":0.72399999999999998,"15min":0.627},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.0600000000000001,"5min":1.2090000000000001,"15min":1.2090000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.77000000000000002,"15min":0.68300000000000005},"min":{"1min":0.372,"5min":0.36299999999999999,"15min":0.35999999999999999},"max":{"1min":1.0089999999999999,"5min":1.296,"15min":1.296},"last":0.59299999999999997}]},{"osd":1,"last update":"Sat Apr 13 07:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.78600000000000003,"5min":0.78900000000000003,"15min":0.69699999999999995},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.33200000000000002},"max":{"1min":1.3,"5min":1.3,"15min":1.3},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.873,"5min":0.84099999999999997,"15min":0.72999999999999998},"min":{"1min":0.44400000000000001,"5min":0.309,"15min":0.28899999999999998},"max":{"1min":1.748,"5min":1.748,"15min":1.748},"last":0.86699999999999999}]},{"osd":3,"last update":"Sat Apr 13 07:32:28 2024","interfaces":[{"interface":"back","average":{"1min":0.88600000000000001,"5min":0.88500000000000001,"15min":0.80600000000000005},"min":{"1min":0.46600000000000003,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.1819999999999999,"5min":1.4510000000000001,"15min":1.4510000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.90500000000000003,"5min":0.89900000000000002,"15min":0.84299999999999997},"min":{"1min":0.53700000000000003,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":1.4830000000000001,"5min":1.4830000000000001,"15min":1.4830000000000001},"last":0.95399999999999996}]},{"osd":4,"last update":"Sat Apr 13 07:33:22 2024","interfaces":[{"interface":"back","average":{"1min":0.91700000000000004,"5min":0.91000000000000003,"15min":0.91400000000000003},"min":{"1min":0.628,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.1200000000000001,"5min":1.341,"15min":1.341},"last":0.747},{"interface":"front","average":{"1min":1.04,"5min":1.0609999999999999,"15min":1.036},"min":{"1min":0.57999999999999996,"5min":0.51800000000000002,"15min":0.51800000000000002},"max":{"1min":1.4279999999999999,"5min":1.52,"15min":1.52},"last":0.97799999999999998}]},{"osd":5,"last update":"Sat Apr 13 07:33:01 2024","interfaces":[{"interface":"back","average":{"1min":1.028,"5min":1.0469999999999999,"15min":1.0569999999999999},"min":{"1min":0.70299999999999996,"5min":0.498,"15min":0.498},"max":{"1min":1.611,"5min":1.611,"15min":1.611},"last":1.012},{"interface":"front","average":{"1min":0.94799999999999995,"5min":0.95399999999999996,"15min":0.96799999999999997},"min":{"1min":0.65700000000000003,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":1.393,"5min":1.456,"15min":1.4850000000000001},"last":0.79100000000000004}]},{"osd":6,"last update":"Sat Apr 13 07:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.052,"5min":1.123,"15min":1.1699999999999999},"min":{"1min":0.61399999999999999,"5min":0.51200000000000001,"15min":0.51200000000000001},"max":{"1min":1.554,"5min":1.788,"15min":1.788},"last":0.90400000000000003},{"interface":"front","average":{"1min":1.1120000000000001,"5min":1.1279999999999999,"15min":1.135},"min":{"1min":0.65000000000000002,"5min":0.59199999999999997,"15min":0.59199999999999997},"max":{"1min":1.9199999999999999,"5min":1.9199999999999999,"15min":1.9199999999999999},"last":1.0589999999999999}]},{"osd":7,"last update":"Sat Apr 13 07:32:45 2024","interfaces":[{"interface":"back","average":{"1min":1.1180000000000001,"5min":1.1439999999999999,"15min":1.1279999999999999},"min":{"1min":0.63100000000000001,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.6419999999999999,"5min":1.651,"15min":1.651},"last":1.036},{"interface":"front","average":{"1min":1.1619999999999999,"5min":1.1919999999999999,"15min":1.1919999999999999},"min":{"1min":0.72199999999999998,"5min":0.68799999999999994,"15min":0.68799999999999994},"max":{"1min":2.008,"5min":2.2130000000000001,"15min":2.2130000000000001},"last":1.109}]}]},{"osd":3,"up_from":26,"seq":111669149798,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27752,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717400,"statfs":{"total":95995035648,"available":95966617600,"internally_reserved":0,"allocated":745472,"data_stored":324667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1592,"internal_metadata":27654600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:33:26 2024","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":0.65000000000000002,"15min":0.66400000000000003},"min":{"1min":0.29399999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":1.081,"5min":1.196,"15min":1.196},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.72099999999999997,"5min":0.72599999999999998,"15min":0.75900000000000001},"min":{"1min":0.32200000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":1.0680000000000001,"5min":1.1599999999999999,"15min":1.206},"last":1.0600000000000001}]},{"osd":1,"last update":"Sat Apr 13 07:33:26 2024","interfaces":[{"interface":"back","average":{"1min":0.72899999999999998,"5min":0.71699999999999997,"15min":0.73199999999999998},"min":{"1min":0.28499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":1.165,"5min":1.165,"15min":1.165},"last":0.92700000000000005},{"interface":"front","average":{"1min":0.76800000000000002,"5min":0.77400000000000002,"15min":0.78600000000000003},"min":{"1min":0.34399999999999997,"5min":0.314,"15min":0.314},"max":{"1min":1.1060000000000001,"5min":1.206,"15min":1.254},"last":0.71899999999999997}]},{"osd":2,"last update":"Sat Apr 13 07:33:26 2024","interfaces":[{"interface":"back","average":{"1min":0.90300000000000002,"5min":0.84099999999999997,"15min":0.83899999999999997},"min":{"1min":0.377,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.3480000000000001,"5min":1.3620000000000001,"15min":1.3620000000000001},"last":0.80300000000000005},{"interface":"front","average":{"1min":0.81100000000000005,"5min":0.83899999999999997,"15min":0.82399999999999995},"min":{"1min":0.35899999999999999,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.095,"5min":1.345,"15min":1.345},"last":1.2410000000000001}]},{"osd":4,"last update":"Sat Apr 13 07:33:12 2024","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":0.92900000000000005,"15min":0.91400000000000003},"min":{"1min":0.61799999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":1.242,"5min":1.347,"15min":1.347},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.012,"5min":1.0069999999999999,"15min":0.96399999999999997},"min":{"1min":0.82799999999999996,"5min":0.495,"15min":0.495},"max":{"1min":1.367,"5min":1.367,"15min":1.389},"last":1.1519999999999999}]},{"osd":5,"last update":"Sat Apr 13 07:33:09 2024","interfaces":[{"interface":"back","average":{"1min":1.077,"5min":1.046,"15min":1.0349999999999999},"min":{"1min":0.76600000000000001,"5min":0.438,"15min":0.438},"max":{"1min":1.274,"5min":1.46,"15min":1.46},"last":0.97299999999999998},{"interface":"front","average":{"1min":1.091,"5min":1.0669999999999999,"15min":1.0589999999999999},"min":{"1min":0.82099999999999995,"5min":0.55300000000000005,"15min":0.55300000000000005},"max":{"1min":1.4490000000000001,"5min":1.4490000000000001,"15min":1.4490000000000001},"last":1.4419999999999999}]},{"osd":6,"last update":"Sat Apr 13 07:32:57 2024","interfaces":[{"interface":"back","average":{"1min":1.115,"5min":1.081,"15min":1.0700000000000001},"min":{"1min":0.92500000000000004,"5min":0.60099999999999998,"15min":0.60099999999999998},"max":{"1min":1.3999999999999999,"5min":1.7010000000000001,"15min":1.7010000000000001},"last":1.385},{"interface":"front","average":{"1min":1.177,"5min":1.0980000000000001,"15min":1.0680000000000001},"min":{"1min":0.93300000000000005,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.3540000000000001,"5min":1.536,"15min":1.536},"last":0.89700000000000002}]},{"osd":7,"last update":"Sat Apr 13 07:32:47 2024","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.127,"15min":1.0820000000000001},"min":{"1min":1.0109999999999999,"5min":0.46300000000000002,"15min":0.46300000000000002},"max":{"1min":1.587,"5min":1.587,"15min":1.587},"last":1.3089999999999999},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.1459999999999999,"15min":1.1299999999999999},"min":{"1min":0.88800000000000001,"5min":0.58799999999999997,"15min":0.58799999999999997},"max":{"1min":1.4470000000000001,"5min":1.611,"15min":1.611},"last":1.0289999999999999}]}]},{"osd":4,"up_from":31,"seq":133143986268,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":31868,"kb_used_data":748,"kb_used_omap":1,"kb_used_meta":31102,"kb_avail":93713284,"statfs":{"total":95995035648,"available":95962402816,"internally_reserved":0,"allocated":765952,"data_stored":326287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":31848902},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.86399999999999999,"5min":0.84899999999999998,"15min":0.86399999999999999},"min":{"1min":0.624,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":1.1180000000000001,"5min":1.4570000000000001,"15min":1.4570000000000001},"last":0.87},{"interface":"front","average":{"1min":0.97499999999999998,"5min":0.95999999999999996,"15min":0.90900000000000003},"min":{"1min":0.54500000000000004,"5min":0.432,"15min":0.432},"max":{"1min":1.4139999999999999,"5min":1.4139999999999999,"15min":1.4139999999999999},"last":1.0760000000000001}]},{"osd":1,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.91400000000000003,"15min":0.89500000000000002},"min":{"1min":0.627,"5min":0.53100000000000003,"15min":0.53100000000000003},"max":{"1min":1.24,"5min":1.333,"15min":1.6819999999999999},"last":0.84699999999999998},{"interface":"front","average":{"1min":0.876,"5min":0.89300000000000002,"15min":0.97699999999999998},"min":{"1min":0.54700000000000004,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":1.2070000000000001,"5min":1.2929999999999999,"15min":1.7190000000000001},"last":1.046}]},{"osd":2,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.96299999999999997,"5min":0.95799999999999996,"15min":0.91200000000000003},"min":{"1min":0.55400000000000005,"5min":0.51400000000000001,"15min":0.51400000000000001},"max":{"1min":1.1910000000000001,"5min":1.2729999999999999,"15min":1.413},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.028,"5min":0.96599999999999997,"15min":0.90500000000000003},"min":{"1min":0.67000000000000004,"5min":0.502,"15min":0.502},"max":{"1min":1.353,"5min":1.48,"15min":1.48},"last":1.018}]},{"osd":3,"last update":"Sat Apr 13 07:33:14 2024","interfaces":[{"interface":"back","average":{"1min":0.95899999999999996,"5min":0.97699999999999998,"15min":0.93000000000000005},"min":{"1min":0.64800000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"max":{"1min":1.258,"5min":1.4910000000000001,"15min":1.7849999999999999},"last":0.91000000000000003},{"interface":"front","average":{"1min":0.96299999999999997,"5min":0.98299999999999998,"15min":0.92500000000000004},"min":{"1min":0.72999999999999998,"5min":0.55900000000000005,"15min":0.55900000000000005},"max":{"1min":1.3140000000000001,"5min":1.524,"15min":1.524},"last":0.93300000000000005}]},{"osd":5,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":0.92900000000000005,"5min":0.92400000000000004,"15min":0.81000000000000005},"min":{"1min":0.38800000000000001,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":1.2310000000000001,"5min":1.5600000000000001,"15min":1.5600000000000001},"last":0.81000000000000005},{"interface":"front","average":{"1min":0.97799999999999998,"5min":0.94199999999999995,"15min":0.75600000000000001},"min":{"1min":0.46500000000000002,"5min":0.35299999999999998,"15min":0.28000000000000003},"max":{"1min":1.371,"5min":1.76,"15min":1.76},"last":1.0960000000000001}]},{"osd":6,"last update":"Sat Apr 13 07:33:03 2024","interfaces":[{"interface":"back","average":{"1min":1.0049999999999999,"5min":1,"15min":0.95399999999999996},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":1.2929999999999999,"5min":1.675,"15min":1.675},"last":0.999},{"interface":"front","average":{"1min":0.98699999999999999,"5min":1.0249999999999999,"15min":0.999},"min":{"1min":0.33800000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":1.337,"5min":1.734,"15min":1.734},"last":0.97699999999999998}]},{"osd":7,"last update":"Sat Apr 13 07:32:51 2024","interfaces":[{"interface":"back","average":{"1min":1.048,"5min":1.034,"15min":0.99199999999999999},"min":{"1min":0.40600000000000003,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":1.3520000000000001,"5min":1.8380000000000001,"15min":1.8380000000000001},"last":1.1519999999999999},{"interface":"front","average":{"1min":1.117,"5min":1.0209999999999999,"15min":0.96899999999999997},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.5569999999999999,"5min":1.5740000000000001,"15min":1.5740000000000001},"last":0.95899999999999996}]}]},{"osd":5,"up_from":39,"seq":167503724626,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":27748,"kb_used_data":728,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93717404,"statfs":{"total":95995035648,"available":95966621696,"internally_reserved":0,"allocated":745472,"data_stored":324898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.97099999999999997,"5min":0.92100000000000004,"15min":0.90000000000000002},"min":{"1min":0.70399999999999996,"5min":0.435,"15min":0.435},"max":{"1min":1.4410000000000001,"5min":1.4410000000000001,"15min":1.4410000000000001},"last":0.99299999999999999},{"interface":"front","average":{"1min":0.91600000000000004,"5min":0.93100000000000005,"15min":0.89100000000000001},"min":{"1min":0.629,"5min":0.60999999999999999,"15min":0.60999999999999999},"max":{"1min":1.1950000000000001,"5min":1.266,"15min":1.266},"last":0.91000000000000003}]},{"osd":1,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.97599999999999998,"5min":0.97399999999999998,"15min":0.91800000000000004},"min":{"1min":0.67500000000000004,"5min":0.56100000000000005,"15min":0.56100000000000005},"max":{"1min":1.3859999999999999,"5min":1.524,"15min":1.524},"last":0.86799999999999999},{"interface":"front","average":{"1min":0.98299999999999998,"5min":0.92600000000000005,"15min":0.92200000000000004},"min":{"1min":0.70899999999999996,"5min":0.50700000000000001,"15min":0.50700000000000001},"max":{"1min":1.2350000000000001,"5min":1.2669999999999999,"15min":1.2669999999999999},"last":1.032}]},{"osd":2,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":1.0229999999999999,"5min":1.002,"15min":0.92400000000000004},"min":{"1min":0.71599999999999997,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":1.2589999999999999,"5min":1.343,"15min":1.343},"last":0.93700000000000006},{"interface":"front","average":{"1min":1.002,"5min":0.98899999999999999,"15min":0.95499999999999996},"min":{"1min":0.65700000000000003,"5min":0.55400000000000005,"15min":0.55400000000000005},"max":{"1min":1.5229999999999999,"5min":1.5229999999999999,"15min":1.5229999999999999},"last":1.1479999999999999}]},{"osd":3,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":1.095,"5min":1.0049999999999999,"15min":0.95299999999999996},"min":{"1min":0.77400000000000002,"5min":0.57299999999999995,"15min":0.57299999999999995},"max":{"1min":1.4790000000000001,"5min":1.4790000000000001,"15min":1.4790000000000001},"last":1.1200000000000001},{"interface":"front","average":{"1min":1.069,"5min":1.038,"15min":0.98799999999999999},"min":{"1min":0.746,"5min":0.58399999999999996,"15min":0.58399999999999996},"max":{"1min":1.4450000000000001,"5min":1.6890000000000001,"15min":1.6890000000000001},"last":1.087}]},{"osd":4,"last update":"Sat Apr 13 07:33:05 2024","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":0.89100000000000001,"15min":0.79900000000000004},"min":{"1min":0.57399999999999995,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.3129999999999999,"5min":1.3460000000000001,"15min":1.3460000000000001},"last":0.70799999999999996},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.997,"15min":0.85099999999999998},"min":{"1min":0.68999999999999995,"5min":0.318,"15min":0.318},"max":{"1min":1.448,"5min":1.7350000000000001,"15min":1.7350000000000001},"last":1.0569999999999999}]},{"osd":6,"last update":"Sat Apr 13 07:32:54 2024","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.0249999999999999,"15min":0.85899999999999999},"min":{"1min":0.64700000000000002,"5min":0.37,"15min":0.37},"max":{"1min":1.5069999999999999,"5min":1.5069999999999999,"15min":1.5069999999999999},"last":0.82499999999999996},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.0529999999999999,"15min":0.85699999999999998},"min":{"1min":0.71699999999999997,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":1.55,"5min":1.7909999999999999,"15min":1.7909999999999999},"last":1.2210000000000001}]},{"osd":7,"last update":"Sat Apr 13 07:32:49 2024","interfaces":[{"interface":"back","average":{"1min":1.0589999999999999,"5min":1.038,"15min":0.95599999999999996},"min":{"1min":0.625,"5min":0.48399999999999999,"15min":0.48399999999999999},"max":{"1min":1.3959999999999999,"5min":1.454,"15min":1.454},"last":0.96899999999999997},{"interface":"front","average":{"1min":1.1930000000000001,"5min":1.1539999999999999,"15min":1.1000000000000001},"min":{"1min":0.74299999999999999,"5min":0.54000000000000004,"15min":0.54000000000000004},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":1.1830000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1710,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-13T07:33:35.176 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-13T07:33:35.176 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-13T07:33:35.176 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-13T07:33:35.177 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:33:35.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:35 smithi203 bash[23480]: audit 2024-04-13T07:33:34.590533+0000 mgr.y (mgr.14373) 159 : audit 0 from='client.14721 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:33:35.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:35 smithi089 bash[31522]: audit 2024-04-13T07:33:34.590533+0000 mgr.y (mgr.14373) 159 : audit 0 from='client.14721 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:33:35.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:35 smithi089 bash[21353]: audit 2024-04-13T07:33:34.590533+0000 mgr.y (mgr.14373) 159 : audit 0 from='client.14721 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:33:36.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:36 smithi203 bash[23480]: cluster 2024-04-13T07:33:34.924643+0000 mgr.y (mgr.14373) 160 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:36.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:36 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:33:36] "GET /metrics HTTP/1.1" 200 33770 "" "Prometheus/2.43.0" 2024-04-13T07:33:36.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:36 smithi089 bash[31522]: cluster 2024-04-13T07:33:34.924643+0000 mgr.y (mgr.14373) 160 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:36.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:36 smithi089 bash[21353]: cluster 2024-04-13T07:33:34.924643+0000 mgr.y (mgr.14373) 160 : cluster 0 pgmap v133: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:38.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:38 smithi203 bash[23480]: cluster 2024-04-13T07:33:36.925255+0000 mgr.y (mgr.14373) 161 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:38.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:38 smithi089 bash[31522]: cluster 2024-04-13T07:33:36.925255+0000 mgr.y (mgr.14373) 161 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:38.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:38 smithi089 bash[21353]: cluster 2024-04-13T07:33:36.925255+0000 mgr.y (mgr.14373) 161 : cluster 0 pgmap v134: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:39.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:39 smithi203 bash[23480]: audit 2024-04-13T07:33:38.934460+0000 mon.a (mon.0) 898 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:39.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:39 smithi089 bash[21353]: audit 2024-04-13T07:33:38.934460+0000 mon.a (mon.0) 898 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:39.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:39 smithi089 bash[31522]: audit 2024-04-13T07:33:38.934460+0000 mon.a (mon.0) 898 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:39.908 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:40.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:40 smithi203 bash[23480]: cluster 2024-04-13T07:33:38.925879+0000 mgr.y (mgr.14373) 162 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:40.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:40 smithi089 bash[31522]: cluster 2024-04-13T07:33:38.925879+0000 mgr.y (mgr.14373) 162 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:40.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:40 smithi089 bash[21353]: cluster 2024-04-13T07:33:38.925879+0000 mgr.y (mgr.14373) 162 : cluster 0 pgmap v135: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:42.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:42 smithi089 bash[31522]: cluster 2024-04-13T07:33:40.927059+0000 mgr.y (mgr.14373) 163 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:42.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:42 smithi089 bash[21353]: cluster 2024-04-13T07:33:40.927059+0000 mgr.y (mgr.14373) 163 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:42.381 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:33:42.381 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2024-04-13T07:33:42.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:42 smithi203 bash[23480]: cluster 2024-04-13T07:33:40.927059+0000 mgr.y (mgr.14373) 163 : cluster 0 pgmap v136: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:43.009 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2024-04-13T07:33:43.009 INFO:teuthology.run_tasks:Running task print... 2024-04-13T07:33:43.019 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2024-04-13T07:33:43.020 INFO:teuthology.run_tasks:Running task parallel... 2024-04-13T07:33:43.086 INFO:teuthology.task.parallel:starting parallel... 2024-04-13T07:33:43.087 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-13T07:33:43.087 INFO:teuthology.task.sequential:In sequential, running task workunit... 2024-04-13T07:33:43.092 INFO:tasks.workunit:Pulling workunits from ref reef 2024-04-13T07:33:43.093 INFO:tasks.workunit:Making a separate scratch dir for every client... 2024-04-13T07:33:43.093 DEBUG:teuthology.orchestra.run.smithi089:> stat -- /home/ubuntu/cephtest/mnt.0 2024-04-13T07:33:43.093 INFO:teuthology.task.parallel:In parallel, running task sequential... 2024-04-13T07:33:43.094 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2024-04-13T07:33:43.094 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi089.front.sepia.ceph.com 2024-04-13T07:33:43.094 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2024-04-13T07:33:43.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-13T07:33:43.100 INFO:teuthology.orchestra.run.smithi089.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2024-04-13T07:33:43.101 DEBUG:teuthology.orchestra.run.smithi089:> mkdir -- /home/ubuntu/cephtest/mnt.0 2024-04-13T07:33:43.148 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2024-04-13T07:33:43.148 DEBUG:teuthology.orchestra.run.smithi089:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2024-04-13T07:33:43.196 INFO:tasks.workunit:timeout=3h 2024-04-13T07:33:43.196 INFO:tasks.workunit:cleanup=True 2024-04-13T07:33:43.196 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch reef https://git.ceph.com/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2024-04-13T07:33:43.247 INFO:tasks.workunit.client.0.smithi089.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2024-04-13T07:33:43.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:43 smithi089 bash[31522]: audit 2024-04-13T07:33:42.388681+0000 mon.a (mon.0) 899 : audit 0 from='client.? 172.21.15.89:0/214583300' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:33:43.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:43 smithi089 bash[21353]: audit 2024-04-13T07:33:42.388681+0000 mon.a (mon.0) 899 : audit 0 from='client.? 172.21.15.89:0/214583300' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:33:43.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:43 smithi203 bash[23480]: audit 2024-04-13T07:33:42.388681+0000 mon.a (mon.0) 899 : audit 0 from='client.? 172.21.15.89:0/214583300' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:33:44.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:44 smithi203 bash[23480]: cluster 2024-04-13T07:33:42.927668+0000 mgr.y (mgr.14373) 164 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:44.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:44 smithi089 bash[21353]: cluster 2024-04-13T07:33:42.927668+0000 mgr.y (mgr.14373) 164 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:44.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:44 smithi089 bash[31522]: cluster 2024-04-13T07:33:42.927668+0000 mgr.y (mgr.14373) 164 : cluster 0 pgmap v137: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:46.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:46 smithi203 bash[23480]: cluster 2024-04-13T07:33:44.928763+0000 mgr.y (mgr.14373) 165 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:46.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:46 smithi089 bash[21353]: cluster 2024-04-13T07:33:44.928763+0000 mgr.y (mgr.14373) 165 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:46.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:46 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:33:46] "GET /metrics HTTP/1.1" 200 33768 "" "Prometheus/2.43.0" 2024-04-13T07:33:46.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:46 smithi089 bash[31522]: cluster 2024-04-13T07:33:44.928763+0000 mgr.y (mgr.14373) 165 : cluster 0 pgmap v138: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:47.839 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:48.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:48 smithi203 bash[23480]: cluster 2024-04-13T07:33:46.929333+0000 mgr.y (mgr.14373) 166 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:48.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:48 smithi089 bash[21353]: cluster 2024-04-13T07:33:46.929333+0000 mgr.y (mgr.14373) 166 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:48.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:48 smithi089 bash[31522]: cluster 2024-04-13T07:33:46.929333+0000 mgr.y (mgr.14373) 166 : cluster 0 pgmap v139: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:50.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:50 smithi089 bash[21353]: cluster 2024-04-13T07:33:48.929904+0000 mgr.y (mgr.14373) 167 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:50.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:50 smithi089 bash[31522]: cluster 2024-04-13T07:33:48.929904+0000 mgr.y (mgr.14373) 167 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:50.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:50 smithi203 bash[23480]: cluster 2024-04-13T07:33:48.929904+0000 mgr.y (mgr.14373) 167 : cluster 0 pgmap v140: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:50.797 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:50 smithi089 bash[21606]: [13/Apr/2024:07:33:50] ENGINE Bus STOPPING 2024-04-13T07:33:50.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:50 smithi089 bash[21606]: [13/Apr/2024:07:33:50] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-13T07:33:50.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:50 smithi089 bash[21606]: [13/Apr/2024:07:33:50] ENGINE Bus STOPPED 2024-04-13T07:33:50.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:50 smithi089 bash[21606]: [13/Apr/2024:07:33:50] ENGINE Bus STARTING 2024-04-13T07:33:50.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:50 smithi089 bash[21606]: [13/Apr/2024:07:33:50] ENGINE Serving on http://:::9283 2024-04-13T07:33:50.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:50 smithi089 bash[21606]: [13/Apr/2024:07:33:50] ENGINE Bus STARTED 2024-04-13T07:33:51.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph config set global log_to_journald false --force' 2024-04-13T07:33:51.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:51 smithi203 bash[23480]: audit 2024-04-13T07:33:50.389355+0000 mon.a (mon.0) 900 : audit 1 from='client.? 172.21.15.89:0/3579534725' entity='client.admin' 2024-04-13T07:33:51.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:51 smithi203 bash[23480]: audit 2024-04-13T07:33:50.417883+0000 mon.a (mon.0) 901 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:33:51.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:51 smithi089 bash[21353]: audit 2024-04-13T07:33:50.389355+0000 mon.a (mon.0) 900 : audit 1 from='client.? 172.21.15.89:0/3579534725' entity='client.admin' 2024-04-13T07:33:51.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:51 smithi089 bash[21353]: audit 2024-04-13T07:33:50.417883+0000 mon.a (mon.0) 901 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:33:51.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:51 smithi089 bash[31522]: audit 2024-04-13T07:33:50.389355+0000 mon.a (mon.0) 900 : audit 1 from='client.? 172.21.15.89:0/3579534725' entity='client.admin' 2024-04-13T07:33:51.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:51 smithi089 bash[31522]: audit 2024-04-13T07:33:50.417883+0000 mon.a (mon.0) 901 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:33:52.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:52 smithi203 bash[23480]: cluster 2024-04-13T07:33:50.931056+0000 mgr.y (mgr.14373) 168 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:52.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:52 smithi089 bash[31522]: cluster 2024-04-13T07:33:50.931056+0000 mgr.y (mgr.14373) 168 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:52.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:52 smithi089 bash[21353]: cluster 2024-04-13T07:33:50.931056+0000 mgr.y (mgr.14373) 168 : cluster 0 pgmap v141: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:54.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:54 smithi203 bash[23480]: cluster 2024-04-13T07:33:52.931667+0000 mgr.y (mgr.14373) 169 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:54.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:54 smithi203 bash[23480]: audit 2024-04-13T07:33:53.934150+0000 mon.a (mon.0) 902 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:54.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:54 smithi089 bash[21353]: cluster 2024-04-13T07:33:52.931667+0000 mgr.y (mgr.14373) 169 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:54.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:54 smithi089 bash[21353]: audit 2024-04-13T07:33:53.934150+0000 mon.a (mon.0) 902 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:54.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:54 smithi089 bash[31522]: cluster 2024-04-13T07:33:52.931667+0000 mgr.y (mgr.14373) 169 : cluster 0 pgmap v142: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:54.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:54 smithi089 bash[31522]: audit 2024-04-13T07:33:53.934150+0000 mon.a (mon.0) 902 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:33:55.267 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:33:56.450 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:33:56] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.43.0" 2024-04-13T07:33:56.450 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[31522]: cluster 2024-04-13T07:33:54.932661+0000 mgr.y (mgr.14373) 170 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:56.450 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[31522]: audit 2024-04-13T07:33:55.956676+0000 mon.a (mon.0) 903 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.450 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[31522]: audit 2024-04-13T07:33:55.961370+0000 mon.a (mon.0) 904 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.450 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[31522]: audit 2024-04-13T07:33:56.169674+0000 mon.a (mon.0) 905 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.450 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[31522]: audit 2024-04-13T07:33:56.180788+0000 mon.a (mon.0) 906 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[31522]: audit 2024-04-13T07:33:56.182674+0000 mon.a (mon.0) 907 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[31522]: audit 2024-04-13T07:33:56.184024+0000 mon.a (mon.0) 908 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[31522]: audit 2024-04-13T07:33:56.192696+0000 mon.a (mon.0) 909 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21353]: cluster 2024-04-13T07:33:54.932661+0000 mgr.y (mgr.14373) 170 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21353]: audit 2024-04-13T07:33:55.956676+0000 mon.a (mon.0) 903 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21353]: audit 2024-04-13T07:33:55.961370+0000 mon.a (mon.0) 904 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21353]: audit 2024-04-13T07:33:56.169674+0000 mon.a (mon.0) 905 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21353]: audit 2024-04-13T07:33:56.180788+0000 mon.a (mon.0) 906 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21353]: audit 2024-04-13T07:33:56.182674+0000 mon.a (mon.0) 907 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:33:56.451 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21353]: audit 2024-04-13T07:33:56.184024+0000 mon.a (mon.0) 908 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:33:56.452 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:56 smithi089 bash[21353]: audit 2024-04-13T07:33:56.192696+0000 mon.a (mon.0) 909 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:56 smithi203 bash[23480]: cluster 2024-04-13T07:33:54.932661+0000 mgr.y (mgr.14373) 170 : cluster 0 pgmap v143: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:56.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:56 smithi203 bash[23480]: audit 2024-04-13T07:33:55.956676+0000 mon.a (mon.0) 903 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:56 smithi203 bash[23480]: audit 2024-04-13T07:33:55.961370+0000 mon.a (mon.0) 904 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:56 smithi203 bash[23480]: audit 2024-04-13T07:33:56.169674+0000 mon.a (mon.0) 905 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:56 smithi203 bash[23480]: audit 2024-04-13T07:33:56.180788+0000 mon.a (mon.0) 906 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:56 smithi203 bash[23480]: audit 2024-04-13T07:33:56.182674+0000 mon.a (mon.0) 907 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:33:56.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:56 smithi203 bash[23480]: audit 2024-04-13T07:33:56.184024+0000 mon.a (mon.0) 908 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:33:56.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:56 smithi203 bash[23480]: audit 2024-04-13T07:33:56.192696+0000 mon.a (mon.0) 909 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:33:56.833 DEBUG:teuthology.orchestra.run.smithi089:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2024-04-13T07:33:56.839 INFO:tasks.workunit.client.0.smithi089.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2024-04-13T07:33:56.843 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-13T07:33:56.843 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2024-04-13T07:33:58.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:33:58 smithi203 bash[23480]: cluster 2024-04-13T07:33:56.933235+0000 mgr.y (mgr.14373) 171 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:58.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:33:58 smithi089 bash[21353]: cluster 2024-04-13T07:33:56.933235+0000 mgr.y (mgr.14373) 171 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:33:58.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:33:58 smithi089 bash[31522]: cluster 2024-04-13T07:33:56.933235+0000 mgr.y (mgr.14373) 171 : cluster 0 pgmap v144: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:00.396 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2024-04-13T07:34:00.472 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2024-04-13T07:34:00.530 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2024-04-13T07:34:00.531 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-13T07:34:00.531 INFO:tasks.workunit.client.0.smithi089.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2024-04-13T07:34:00.573 INFO:tasks.workunit.client.0.smithi089.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2024-04-13T07:34:00.576 DEBUG:teuthology.orchestra.run.smithi089:> set -ex 2024-04-13T07:34:00.577 DEBUG:teuthology.orchestra.run.smithi089:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2024-04-13T07:34:00.628 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2024-04-13T07:34:00.629 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2024-04-13T07:34:00.629 DEBUG:teuthology.orchestra.run.smithi089:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=reef TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2024-04-13T07:34:00.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:00 smithi089 bash[21353]: cluster 2024-04-13T07:33:58.933850+0000 mgr.y (mgr.14373) 172 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:00.673 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:00 smithi089 bash[31522]: cluster 2024-04-13T07:33:58.933850+0000 mgr.y (mgr.14373) 172 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:00.679 INFO:tasks.workunit.client.0.smithi089.stderr:+ export RBD_FORCE_ALLOW_V1=1 2024-04-13T07:34:00.679 INFO:tasks.workunit.client.0.smithi089.stderr:+ TMPDIR=/tmp/rbd_import_export_69261 2024-04-13T07:34:00.679 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm -rf /tmp/rbd_import_export_69261 2024-04-13T07:34:00.680 INFO:tasks.workunit.client.0.smithi089.stderr:+ mkdir /tmp/rbd_import_export_69261 2024-04-13T07:34:00.680 INFO:tasks.workunit.client.0.smithi089.stderr:+ trap rm -rf /tmp/rbd_import_export_69261 INT TERM EXIT 2024-04-13T07:34:00.681 INFO:tasks.workunit.client.0.smithi089.stderr:+ mkdir foo.69261 2024-04-13T07:34:00.681 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import foo.69261 foo.dir 2024-04-13T07:34:00.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:00 smithi203 bash[23480]: cluster 2024-04-13T07:33:58.933850+0000 mgr.y (mgr.14373) 172 : cluster 0 pgmap v145: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:01.040 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: cannot import a directory 2024-04-13T07:34:01.040 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 0% complete...failed. 2024-04-13T07:34:01.040 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: import failed: (21) Is a directory 2024-04-13T07:34:01.141 INFO:tasks.workunit.client.0.smithi089.stderr:+ true 2024-04-13T07:34:01.142 INFO:tasks.workunit.client.0.smithi089.stderr:+ rmdir foo.69261 2024-04-13T07:34:01.177 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_69261/img bs=1k count=1 seek=10 2024-04-13T07:34:01.178 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records in 2024-04-13T07:34:01.178 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records out 2024-04-13T07:34:01.179 INFO:tasks.workunit.client.0.smithi089.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 0.000207485 s, 4.9 MB/s 2024-04-13T07:34:01.179 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_69261/img bs=1k count=10 seek=100 2024-04-13T07:34:01.179 INFO:tasks.workunit.client.0.smithi089.stderr:10+0 records in 2024-04-13T07:34:01.179 INFO:tasks.workunit.client.0.smithi089.stderr:10+0 records out 2024-04-13T07:34:01.179 INFO:tasks.workunit.client.0.smithi089.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000142252 s, 72.0 MB/s 2024-04-13T07:34:01.179 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_69261/img bs=1k count=100 seek=1000 2024-04-13T07:34:01.180 INFO:tasks.workunit.client.0.smithi089.stderr:58+1 records in 2024-04-13T07:34:01.180 INFO:tasks.workunit.client.0.smithi089.stderr:58+1 records out 2024-04-13T07:34:01.180 INFO:tasks.workunit.client.0.smithi089.stderr:59904 bytes (60 kB, 58 KiB) copied, 0.000311861 s, 192 MB/s 2024-04-13T07:34:01.181 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_69261/img bs=1k seek=10000 2024-04-13T07:34:01.182 INFO:tasks.workunit.client.0.smithi089.stderr:134+1 records in 2024-04-13T07:34:01.182 INFO:tasks.workunit.client.0.smithi089.stderr:134+1 records out 2024-04-13T07:34:01.182 INFO:tasks.workunit.client.0.smithi089.stderr:138208 bytes (138 kB, 135 KiB) copied, 0.000756456 s, 183 MB/s 2024-04-13T07:34:01.182 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_69261/img bs=1k seek=100000 2024-04-13T07:34:01.184 INFO:tasks.workunit.client.0.smithi089.stderr:58+1 records in 2024-04-13T07:34:01.184 INFO:tasks.workunit.client.0.smithi089.stderr:58+1 records out 2024-04-13T07:34:01.184 INFO:tasks.workunit.client.0.smithi089.stderr:59904 bytes (60 kB, 58 KiB) copied, 0.00032565 s, 184 MB/s 2024-04-13T07:34:01.184 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_69261/img bs=1k seek=1000000 2024-04-13T07:34:01.185 INFO:tasks.workunit.client.0.smithi089.stderr:178+1 records in 2024-04-13T07:34:01.185 INFO:tasks.workunit.client.0.smithi089.stderr:178+1 records out 2024-04-13T07:34:01.185 INFO:tasks.workunit.client.0.smithi089.stderr:182728 bytes (183 kB, 178 KiB) copied, 0.000783693 s, 233 MB/s 2024-04-13T07:34:01.185 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm testimg 2024-04-13T07:34:01.240 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 0% complete...failed. 2024-04-13T07:34:01.241 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: delete error: (2) No such file or directory 2024-04-13T07:34:01.246 INFO:tasks.workunit.client.0.smithi089.stderr:+ true 2024-04-13T07:34:01.246 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format /tmp/rbd_import_export_69261/img testimg 2024-04-13T07:34:01.259 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2024-04-13T07:34:02.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:02 smithi203 bash[23480]: cluster 2024-04-13T07:34:00.934892+0000 mgr.y (mgr.14373) 173 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:02.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:02 smithi203 bash[23480]: cluster 2024-04-13T07:34:01.428747+0000 mon.a (mon.0) 910 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-04-13T07:34:02.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:02 smithi089 bash[21353]: cluster 2024-04-13T07:34:00.934892+0000 mgr.y (mgr.14373) 173 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:02.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:02 smithi089 bash[21353]: cluster 2024-04-13T07:34:01.428747+0000 mon.a (mon.0) 910 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-04-13T07:34:02.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:02 smithi089 bash[31522]: cluster 2024-04-13T07:34:00.934892+0000 mgr.y (mgr.14373) 173 : cluster 0 pgmap v146: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:02.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:02 smithi089 bash[31522]: cluster 2024-04-13T07:34:01.428747+0000 mon.a (mon.0) 910 : cluster 0 osdmap e73: 8 total, 8 up, 8 in 2024-04-13T07:34:03.031 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2024-04-13T07:34:03.031 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:03.035 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export testimg /tmp/rbd_import_export_69261/img2 2024-04-13T07:34:03.236 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:03.242 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export testimg - 2024-04-13T07:34:03.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:03 smithi203 bash[23480]: cluster 2024-04-13T07:34:02.441879+0000 mon.a (mon.0) 911 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-04-13T07:34:03.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:03 smithi089 bash[31522]: cluster 2024-04-13T07:34:02.441879+0000 mon.a (mon.0) 911 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-04-13T07:34:03.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:03 smithi089 bash[21353]: cluster 2024-04-13T07:34:02.441879+0000 mon.a (mon.0) 911 : cluster 0 osdmap e74: 8 total, 8 up, 8 in 2024-04-13T07:34:03.937 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:03.942 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm testimg 2024-04-13T07:34:04.103 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-04-13T07:34:04.107 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp /tmp/rbd_import_export_69261/img /tmp/rbd_import_export_69261/img2 2024-04-13T07:34:04.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:04 smithi203 bash[23480]: cluster 2024-04-13T07:34:02.935485+0000 mgr.y (mgr.14373) 174 : cluster 0 pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:04.789 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp /tmp/rbd_import_export_69261/img /tmp/rbd_import_export_69261/img3 2024-04-13T07:34:04.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:04 smithi089 bash[31522]: cluster 2024-04-13T07:34:02.935485+0000 mgr.y (mgr.14373) 174 : cluster 0 pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:04.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:04 smithi089 bash[21353]: cluster 2024-04-13T07:34:02.935485+0000 mgr.y (mgr.14373) 174 : cluster 0 pgmap v149: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:05.293 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img2 /tmp/rbd_import_export_69261/img3 2024-04-13T07:34:05.483 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format - testimg 2024-04-13T07:34:05.979 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:05.979 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:05.983 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export testimg /tmp/rbd_import_export_69261/img2 2024-04-13T07:34:06.023 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:34:06.181 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:06.185 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export testimg - 2024-04-13T07:34:06.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:34:06 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:34:06] "GET /metrics HTTP/1.1" 200 33771 "" "Prometheus/2.43.0" 2024-04-13T07:34:06.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:06 smithi203 bash[23480]: cluster 2024-04-13T07:34:04.936563+0000 mgr.y (mgr.14373) 175 : cluster 0 pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 59 KiB/s wr, 38 op/s 2024-04-13T07:34:06.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:06 smithi089 bash[31522]: cluster 2024-04-13T07:34:04.936563+0000 mgr.y (mgr.14373) 175 : cluster 0 pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 59 KiB/s wr, 38 op/s 2024-04-13T07:34:06.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:06 smithi089 bash[21353]: cluster 2024-04-13T07:34:04.936563+0000 mgr.y (mgr.14373) 175 : cluster 0 pgmap v150: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 59 KiB/s wr, 38 op/s 2024-04-13T07:34:06.869 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:06.874 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm testimg 2024-04-13T07:34:06.994 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2024-04-13T07:34:06.997 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp /tmp/rbd_import_export_69261/img /tmp/rbd_import_export_69261/img2 2024-04-13T07:34:07.648 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp /tmp/rbd_import_export_69261/img /tmp/rbd_import_export_69261/img3 2024-04-13T07:34:08.153 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img /tmp/rbd_import_export_69261/img2 /tmp/rbd_import_export_69261/img3 2024-04-13T07:34:08.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:08 smithi203 bash[23480]: cluster 2024-04-13T07:34:06.936976+0000 mgr.y (mgr.14373) 176 : cluster 0 pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 59 KiB/s wr, 38 op/s 2024-04-13T07:34:08.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:08 smithi203 bash[23480]: cluster 2024-04-13T07:34:07.901786+0000 mon.a (mon.0) 912 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-04-13T07:34:08.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:08 smithi089 bash[21353]: cluster 2024-04-13T07:34:06.936976+0000 mgr.y (mgr.14373) 176 : cluster 0 pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 59 KiB/s wr, 38 op/s 2024-04-13T07:34:08.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:08 smithi089 bash[21353]: cluster 2024-04-13T07:34:07.901786+0000 mon.a (mon.0) 912 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-04-13T07:34:08.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:08 smithi089 bash[31522]: cluster 2024-04-13T07:34:06.936976+0000 mgr.y (mgr.14373) 176 : cluster 0 pgmap v151: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 139 KiB/s rd, 59 KiB/s wr, 38 op/s 2024-04-13T07:34:08.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:08 smithi089 bash[31522]: cluster 2024-04-13T07:34:07.901786+0000 mon.a (mon.0) 912 : cluster 0 osdmap e75: 8 total, 8 up, 8 in 2024-04-13T07:34:08.966 INFO:tasks.workunit.client.0.smithi089.stderr:+ + rbdgrep -q export-format 2024-04-13T07:34:08.966 INFO:tasks.workunit.client.0.smithi089.stderr: help export 2024-04-13T07:34:08.994 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_69261/img bs=1k count=10 seek=100 2024-04-13T07:34:08.995 INFO:tasks.workunit.client.0.smithi089.stderr:10+0 records in 2024-04-13T07:34:08.995 INFO:tasks.workunit.client.0.smithi089.stderr:10+0 records out 2024-04-13T07:34:08.995 INFO:tasks.workunit.client.0.smithi089.stderr:10240 bytes (10 kB, 10 KiB) copied, 0.000136669 s, 74.9 MB/s 2024-04-13T07:34:08.995 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format /tmp/rbd_import_export_69261/img testimg 2024-04-13T07:34:09.074 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:09.075 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:09.078 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap create testimg@snap 2024-04-13T07:34:09.481 INFO:tasks.workunit.client.0.smithi089.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-13T07:34:09.496 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd image-meta set testimg key1 value1 2024-04-13T07:34:09.567 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd image-meta list testimg 2024-04-13T07:34:09.615 INFO:tasks.workunit.client.0.smithi089.stderr:+ IMAGEMETA_BEFORE=There is 1 metadatum on this image: 2024-04-13T07:34:09.616 INFO:tasks.workunit.client.0.smithi089.stderr: 2024-04-13T07:34:09.616 INFO:tasks.workunit.client.0.smithi089.stderr:Key Value 2024-04-13T07:34:09.616 INFO:tasks.workunit.client.0.smithi089.stderr:key1 value1 2024-04-13T07:34:09.616 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69261/img_v2 2024-04-13T07:34:09.666 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:09.670 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69261/img_v2 testimg_import 2024-04-13T07:34:09.747 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:09 smithi089 bash[31522]: audit 2024-04-13T07:34:08.935239+0000 mon.a (mon.0) 913 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:09.747 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:09 smithi089 bash[21353]: audit 2024-04-13T07:34:08.935239+0000 mon.a (mon.0) 913 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:09.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:09 smithi203 bash[23480]: audit 2024-04-13T07:34:08.935239+0000 mon.a (mon.0) 913 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:10.495 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-13T07:34:10.500 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg_import 2024-04-13T07:34:10.542 INFO:tasks.workunit.client.0.smithi089.stdout:rbd image 'testimg_import': 2024-04-13T07:34:10.542 INFO:tasks.workunit.client.0.smithi089.stdout: size 110 KiB in 1 objects 2024-04-13T07:34:10.542 INFO:tasks.workunit.client.0.smithi089.stdout: order 22 (4 MiB objects) 2024-04-13T07:34:10.542 INFO:tasks.workunit.client.0.smithi089.stdout: snapshot_count: 1 2024-04-13T07:34:10.542 INFO:tasks.workunit.client.0.smithi089.stdout: id: 39f337839c82 2024-04-13T07:34:10.542 INFO:tasks.workunit.client.0.smithi089.stdout: block_name_prefix: rbd_data.39f337839c82 2024-04-13T07:34:10.542 INFO:tasks.workunit.client.0.smithi089.stdout: format: 2 2024-04-13T07:34:10.543 INFO:tasks.workunit.client.0.smithi089.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-04-13T07:34:10.543 INFO:tasks.workunit.client.0.smithi089.stdout: op_features: 2024-04-13T07:34:10.543 INFO:tasks.workunit.client.0.smithi089.stdout: flags: 2024-04-13T07:34:10.543 INFO:tasks.workunit.client.0.smithi089.stdout: create_timestamp: Sat Apr 13 07:34:09 2024 2024-04-13T07:34:10.543 INFO:tasks.workunit.client.0.smithi089.stdout: access_timestamp: Sat Apr 13 07:34:09 2024 2024-04-13T07:34:10.543 INFO:tasks.workunit.client.0.smithi089.stdout: modify_timestamp: Sat Apr 13 07:34:09 2024 2024-04-13T07:34:10.546 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg_import@snap 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout:rbd image 'testimg_import': 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout: size 110 KiB in 1 objects 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout: order 22 (4 MiB objects) 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout: snapshot_count: 1 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout: id: 39f337839c82 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout: block_name_prefix: rbd_data.39f337839c82 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout: format: 2 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2024-04-13T07:34:10.589 INFO:tasks.workunit.client.0.smithi089.stdout: op_features: 2024-04-13T07:34:10.590 INFO:tasks.workunit.client.0.smithi089.stdout: flags: 2024-04-13T07:34:10.590 INFO:tasks.workunit.client.0.smithi089.stdout: create_timestamp: Sat Apr 13 07:34:09 2024 2024-04-13T07:34:10.590 INFO:tasks.workunit.client.0.smithi089.stdout: access_timestamp: Sat Apr 13 07:34:09 2024 2024-04-13T07:34:10.590 INFO:tasks.workunit.client.0.smithi089.stdout: modify_timestamp: Sat Apr 13 07:34:09 2024 2024-04-13T07:34:10.590 INFO:tasks.workunit.client.0.smithi089.stdout: protected: False 2024-04-13T07:34:10.592 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd image-meta list testimg_import 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr:+ IMAGEMETA_AFTER=There is 1 metadatum on this image: 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr: 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr:Key Value 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr:key1 value1 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ There is 1 metadatum on this image: 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr: 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr:Key Value 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr:key1 value1 = There is 1 metadatum on this image: 2024-04-13T07:34:10.640 INFO:tasks.workunit.client.0.smithi089.stderr: 2024-04-13T07:34:10.641 INFO:tasks.workunit.client.0.smithi089.stderr:Key Value 2024-04-13T07:34:10.641 INFO:tasks.workunit.client.0.smithi089.stderr:key1 value1 ] 2024-04-13T07:34:10.641 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export testimg_import /tmp/rbd_import_export_69261/img_import 2024-04-13T07:34:10.679 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 100% complete...done. 2024-04-13T07:34:10.683 INFO:tasks.workunit.client.0.smithi089.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69261/img /tmp/rbd_import_export_69261/img_import 2024-04-13T07:34:10.683 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp -l /tmp/rbd_import_export_69261/img /tmp/rbd_import_export_69261/img_import 2024-04-13T07:34:10.684 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/img --format %b 2024-04-13T07:34:10.685 INFO:tasks.workunit.client.0.smithi089.stderr:+ origsize=24 2024-04-13T07:34:10.685 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/img_import --format %b 2024-04-13T07:34:10.686 INFO:tasks.workunit.client.0.smithi089.stderr:+ exportsize=224 2024-04-13T07:34:10.686 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=200 2024-04-13T07:34:10.686 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=200 2024-04-13T07:34:10.686 INFO:tasks.workunit.client.0.smithi089.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2024-04-13T07:34:10.686 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_69261/img_snap 2024-04-13T07:34:10.728 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 100% complete...done. 2024-04-13T07:34:10.732 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_69261/img_snap_import 2024-04-13T07:34:10.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:10 smithi203 bash[23480]: cluster 2024-04-13T07:34:08.937517+0000 mgr.y (mgr.14373) 177 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 63 KiB/s wr, 40 op/s 2024-04-13T07:34:10.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:10 smithi203 bash[23480]: cluster 2024-04-13T07:34:09.468232+0000 mon.a (mon.0) 914 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-04-13T07:34:10.773 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 100% complete...done. 2024-04-13T07:34:10.777 INFO:tasks.workunit.client.0.smithi089.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69261/img_snap /tmp/rbd_import_export_69261/img_snap_import 2024-04-13T07:34:10.777 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp -l /tmp/rbd_import_export_69261/img_snap /tmp/rbd_import_export_69261/img_snap_import 2024-04-13T07:34:10.778 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/img_snap --format %b 2024-04-13T07:34:10.779 INFO:tasks.workunit.client.0.smithi089.stderr:+ origsize=224 2024-04-13T07:34:10.779 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/img_snap_import --format %b 2024-04-13T07:34:10.779 INFO:tasks.workunit.client.0.smithi089.stderr:+ exportsize=224 2024-04-13T07:34:10.780 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:10.780 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:10.780 INFO:tasks.workunit.client.0.smithi089.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-13T07:34:10.780 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img_v2 2024-04-13T07:34:10.780 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img_import 2024-04-13T07:34:10.780 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img_snap 2024-04-13T07:34:10.781 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img_snap_import 2024-04-13T07:34:10.781 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap rm testimg_import@snap 2024-04-13T07:34:10.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:10 smithi089 bash[31522]: cluster 2024-04-13T07:34:08.937517+0000 mgr.y (mgr.14373) 177 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 63 KiB/s wr, 40 op/s 2024-04-13T07:34:10.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:10 smithi089 bash[31522]: cluster 2024-04-13T07:34:09.468232+0000 mon.a (mon.0) 914 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-04-13T07:34:10.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:10 smithi089 bash[21353]: cluster 2024-04-13T07:34:08.937517+0000 mgr.y (mgr.14373) 177 : cluster 0 pgmap v153: 73 pgs: 73 active+clean; 579 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 148 KiB/s rd, 63 KiB/s wr, 40 op/s 2024-04-13T07:34:10.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:10 smithi089 bash[21353]: cluster 2024-04-13T07:34:09.468232+0000 mon.a (mon.0) 914 : cluster 0 osdmap e76: 8 total, 8 up, 8 in 2024-04-13T07:34:11.484 INFO:tasks.workunit.client.0.smithi089.stderr: Removing snap: 100% complete...done. 2024-04-13T07:34:11.487 INFO:teuthology.orchestra.run.smithi089.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:34:11.496 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg_import 2024-04-13T07:34:11.621 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:11.625 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap rm testimg@snap 2024-04-13T07:34:11.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:11 smithi203 bash[23480]: cluster 2024-04-13T07:34:10.474858+0000 mon.a (mon.0) 915 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-04-13T07:34:11.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:11 smithi089 bash[31522]: cluster 2024-04-13T07:34:10.474858+0000 mon.a (mon.0) 915 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-04-13T07:34:11.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:11 smithi089 bash[21353]: cluster 2024-04-13T07:34:10.474858+0000 mon.a (mon.0) 915 : cluster 0 osdmap e77: 8 total, 8 up, 8 in 2024-04-13T07:34:12.158 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch ps' 2024-04-13T07:34:12.543 INFO:tasks.workunit.client.0.smithi089.stderr: Removing snap: 100% complete...done. 2024-04-13T07:34:12.559 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm testimg 2024-04-13T07:34:12.701 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:12.706 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --order 20 /tmp/rbd_import_export_69261/img testimg 2024-04-13T07:34:12.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: cluster 2024-04-13T07:34:10.938152+0000 mgr.y (mgr.14373) 178 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 799 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 88 KiB/s wr, 138 op/s 2024-04-13T07:34:12.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: audit 2024-04-13T07:34:11.477317+0000 mgr.y (mgr.14373) 179 : audit 0 from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:34:12.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: cephadm 2024-04-13T07:34:11.478966+0000 mgr.y (mgr.14373) 180 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:34:12.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: cluster 2024-04-13T07:34:11.481790+0000 mon.a (mon.0) 916 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-04-13T07:34:12.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: audit 2024-04-13T07:34:11.493628+0000 mon.a (mon.0) 917 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:12.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: audit 2024-04-13T07:34:11.529036+0000 mon.a (mon.0) 918 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:34:12.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: audit 2024-04-13T07:34:11.530679+0000 mon.a (mon.0) 919 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:34:12.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: audit 2024-04-13T07:34:11.531421+0000 mon.a (mon.0) 920 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:34:12.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: audit 2024-04-13T07:34:11.538934+0000 mon.a (mon.0) 921 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:12.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:12 smithi203 bash[23480]: cephadm 2024-04-13T07:34:11.596266+0000 mgr.y (mgr.14373) 181 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:34:12.795 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:12.796 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --order is deprecated, use --object-size 2024-04-13T07:34:12.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: cluster 2024-04-13T07:34:10.938152+0000 mgr.y (mgr.14373) 178 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 799 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 88 KiB/s wr, 138 op/s 2024-04-13T07:34:12.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: audit 2024-04-13T07:34:11.477317+0000 mgr.y (mgr.14373) 179 : audit 0 from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:34:12.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: cephadm 2024-04-13T07:34:11.478966+0000 mgr.y (mgr.14373) 180 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:34:12.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: cluster 2024-04-13T07:34:11.481790+0000 mon.a (mon.0) 916 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-04-13T07:34:12.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: audit 2024-04-13T07:34:11.493628+0000 mon.a (mon.0) 917 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:12.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: audit 2024-04-13T07:34:11.529036+0000 mon.a (mon.0) 918 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:34:12.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: audit 2024-04-13T07:34:11.530679+0000 mon.a (mon.0) 919 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: audit 2024-04-13T07:34:11.531421+0000 mon.a (mon.0) 920 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: audit 2024-04-13T07:34:11.538934+0000 mon.a (mon.0) 921 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[31522]: cephadm 2024-04-13T07:34:11.596266+0000 mgr.y (mgr.14373) 181 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: cluster 2024-04-13T07:34:10.938152+0000 mgr.y (mgr.14373) 178 : cluster 0 pgmap v156: 73 pgs: 73 active+clean; 799 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 251 KiB/s rd, 88 KiB/s wr, 138 op/s 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: audit 2024-04-13T07:34:11.477317+0000 mgr.y (mgr.14373) 179 : audit 0 from='client.14883 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: cephadm 2024-04-13T07:34:11.478966+0000 mgr.y (mgr.14373) 180 : cephadm 1 Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: cluster 2024-04-13T07:34:11.481790+0000 mon.a (mon.0) 916 : cluster 0 osdmap e78: 8 total, 8 up, 8 in 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: audit 2024-04-13T07:34:11.493628+0000 mon.a (mon.0) 917 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: audit 2024-04-13T07:34:11.529036+0000 mon.a (mon.0) 918 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:34:12.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: audit 2024-04-13T07:34:11.530679+0000 mon.a (mon.0) 919 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:34:12.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: audit 2024-04-13T07:34:11.531421+0000 mon.a (mon.0) 920 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:34:12.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: audit 2024-04-13T07:34:11.538934+0000 mon.a (mon.0) 921 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:12.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:12 smithi089 bash[21353]: cephadm 2024-04-13T07:34:11.596266+0000 mgr.y (mgr.14373) 181 : cephadm 1 Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:5f22a4974d6492df1d519913f5251731f043c582 2024-04-13T07:34:12.799 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69261/img_v2 2024-04-13T07:34:12.852 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:12.856 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69261/img_v2 testimg_import 2024-04-13T07:34:12.951 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-13T07:34:12.956 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg_import 2024-04-13T07:34:12.956 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep order 2024-04-13T07:34:12.956 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $2} 2024-04-13T07:34:12.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep 20 2024-04-13T07:34:13.014 INFO:tasks.workunit.client.0.smithi089.stdout:20 2024-04-13T07:34:13.014 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img_v2 2024-04-13T07:34:13.014 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg_import 2024-04-13T07:34:13.166 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:13.171 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg 2024-04-13T07:34:13.338 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:13.343 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_69261/img testimg 2024-04-13T07:34:13.416 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:13.421 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg 2024-04-13T07:34:13.421 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep features 2024-04-13T07:34:13.474 INFO:tasks.workunit.client.0.smithi089.stderr:+ FEATURES_BEFORE= features: layering 2024-04-13T07:34:13.474 INFO:tasks.workunit.client.0.smithi089.stderr: op_features: 2024-04-13T07:34:13.474 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69261/img_v2 2024-04-13T07:34:13.524 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:13.528 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69261/img_v2 testimg_import 2024-04-13T07:34:13.606 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-13T07:34:13.611 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg_import 2024-04-13T07:34:13.611 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep features 2024-04-13T07:34:13.657 INFO:tasks.workunit.client.0.smithi089.stderr:+ FEATURES_AFTER= features: layering 2024-04-13T07:34:13.657 INFO:tasks.workunit.client.0.smithi089.stderr: op_features: 2024-04-13T07:34:13.657 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ features: layering 2024-04-13T07:34:13.657 INFO:tasks.workunit.client.0.smithi089.stderr: op_features: != features: layering 2024-04-13T07:34:13.657 INFO:tasks.workunit.client.0.smithi089.stderr: op_features: ] 2024-04-13T07:34:13.657 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img_v2 2024-04-13T07:34:13.657 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg_import 2024-04-13T07:34:13.770 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:13.773 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg 2024-04-13T07:34:13.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:13 smithi089 bash[21353]: cluster 2024-04-13T07:34:12.544720+0000 mon.a (mon.0) 922 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-04-13T07:34:13.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:13 smithi089 bash[31522]: cluster 2024-04-13T07:34:12.544720+0000 mon.a (mon.0) 922 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-04-13T07:34:13.885 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:13.888 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_69261/img testimg 2024-04-13T07:34:13.966 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:13.970 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69261/img_v2 2024-04-13T07:34:14.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:13 smithi203 bash[23480]: cluster 2024-04-13T07:34:12.544720+0000 mon.a (mon.0) 922 : cluster 0 osdmap e79: 8 total, 8 up, 8 in 2024-04-13T07:34:14.015 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:14.018 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69261/img_v2 testimg_import 2024-04-13T07:34:14.109 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-13T07:34:14.113 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg_import 2024-04-13T07:34:14.113 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep stripe unit 2024-04-13T07:34:14.113 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep -Ei (4 KiB|4096) 2024-04-13T07:34:14.161 INFO:tasks.workunit.client.0.smithi089.stdout: stripe unit: 4 KiB 2024-04-13T07:34:14.162 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg_import 2024-04-13T07:34:14.162 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep stripe count 2024-04-13T07:34:14.162 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $3} 2024-04-13T07:34:14.162 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep 1000 2024-04-13T07:34:14.207 INFO:tasks.workunit.client.0.smithi089.stdout:1000 2024-04-13T07:34:14.207 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/img_v2 2024-04-13T07:34:14.207 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg_import 2024-04-13T07:34:14.356 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-04-13T07:34:14.360 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg 2024-04-13T07:34:14.478 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2024-04-13T07:34:14.481 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_69261/img testimg 2024-04-13T07:34:14.551 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:14.554 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap create testimg@snap1 2024-04-13T07:34:14.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:14 smithi089 bash[21353]: cluster 2024-04-13T07:34:12.938551+0000 mgr.y (mgr.14373) 182 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 799 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 377 KiB/s rd, 132 KiB/s wr, 208 op/s 2024-04-13T07:34:14.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:14 smithi089 bash[31522]: cluster 2024-04-13T07:34:12.938551+0000 mgr.y (mgr.14373) 182 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 799 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 377 KiB/s rd, 132 KiB/s wr, 208 op/s 2024-04-13T07:34:15.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:14 smithi203 bash[23480]: cluster 2024-04-13T07:34:12.938551+0000 mgr.y (mgr.14373) 182 : cluster 0 pgmap v159: 73 pgs: 73 active+clean; 799 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 377 KiB/s rd, 132 KiB/s wr, 208 op/s 2024-04-13T07:34:15.572 INFO:tasks.workunit.client.0.smithi089.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-13T07:34:15.585 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap create testimg@snap2 2024-04-13T07:34:16.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:34:16 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:34:16] "GET /metrics HTTP/1.1" 200 33808 "" "Prometheus/2.43.0" 2024-04-13T07:34:16.591 INFO:tasks.workunit.client.0.smithi089.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2024-04-13T07:34:16.606 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap protect testimg@snap2 2024-04-13T07:34:16.667 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_69261/snap_protect 2024-04-13T07:34:16.720 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:16.724 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_69261/snap_protect testimg_import 2024-04-13T07:34:16.872 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:34:16.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:16 smithi089 bash[31522]: cluster 2024-04-13T07:34:14.939584+0000 mgr.y (mgr.14373) 183 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 597 KiB/s rd, 130 KiB/s wr, 574 op/s 2024-04-13T07:34:16.895 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:16 smithi089 bash[31522]: cluster 2024-04-13T07:34:15.564792+0000 mon.a (mon.0) 923 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-04-13T07:34:16.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:16 smithi089 bash[21353]: cluster 2024-04-13T07:34:14.939584+0000 mgr.y (mgr.14373) 183 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 597 KiB/s rd, 130 KiB/s wr, 574 op/s 2024-04-13T07:34:16.896 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:16 smithi089 bash[21353]: cluster 2024-04-13T07:34:15.564792+0000 mon.a (mon.0) 923 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-04-13T07:34:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:16 smithi203 bash[23480]: cluster 2024-04-13T07:34:14.939584+0000 mgr.y (mgr.14373) 183 : cluster 0 pgmap v160: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 597 KiB/s rd, 130 KiB/s wr, 574 op/s 2024-04-13T07:34:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:16 smithi203 bash[23480]: cluster 2024-04-13T07:34:15.564792+0000 mon.a (mon.0) 923 : cluster 0 osdmap e80: 8 total, 8 up, 8 in 2024-04-13T07:34:17.949 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2024-04-13T07:34:17.956 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg_import@snap1 2024-04-13T07:34:17.956 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep protected: False 2024-04-13T07:34:18.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:17 smithi203 bash[23480]: cluster 2024-04-13T07:34:16.567752+0000 mon.a (mon.0) 924 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-04-13T07:34:18.015 INFO:tasks.workunit.client.0.smithi089.stdout: protected: False 2024-04-13T07:34:18.015 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info testimg_import@snap2 2024-04-13T07:34:18.015 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep protected: True 2024-04-13T07:34:18.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:17 smithi089 bash[21353]: cluster 2024-04-13T07:34:16.567752+0000 mon.a (mon.0) 924 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-04-13T07:34:18.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:17 smithi089 bash[31522]: cluster 2024-04-13T07:34:16.567752+0000 mon.a (mon.0) 924 : cluster 0 osdmap e81: 8 total, 8 up, 8 in 2024-04-13T07:34:18.069 INFO:tasks.workunit.client.0.smithi089.stdout: protected: True 2024-04-13T07:34:18.070 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/snap_protect 2024-04-13T07:34:18.070 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap unprotect testimg@snap2 2024-04-13T07:34:18.126 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap unprotect testimg_import@snap2 2024-04-13T07:34:18.186 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap purge testimg 2024-04-13T07:34:18.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:18 smithi089 bash[31522]: cluster 2024-04-13T07:34:16.940048+0000 mgr.y (mgr.14373) 184 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 323 KiB/s rd, 34 KiB/s wr, 424 op/s 2024-04-13T07:34:18.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:18 smithi089 bash[31522]: cluster 2024-04-13T07:34:17.586972+0000 mon.a (mon.0) 925 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-04-13T07:34:18.840 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:18 smithi089 bash[31522]: cluster 2024-04-13T07:34:17.901508+0000 mon.a (mon.0) 926 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-04-13T07:34:18.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:18 smithi089 bash[21353]: cluster 2024-04-13T07:34:16.940048+0000 mgr.y (mgr.14373) 184 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 323 KiB/s rd, 34 KiB/s wr, 424 op/s 2024-04-13T07:34:18.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:18 smithi089 bash[21353]: cluster 2024-04-13T07:34:17.586972+0000 mon.a (mon.0) 925 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-04-13T07:34:18.840 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:18 smithi089 bash[21353]: cluster 2024-04-13T07:34:17.901508+0000 mon.a (mon.0) 926 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-04-13T07:34:19.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:18 smithi203 bash[23480]: cluster 2024-04-13T07:34:16.940048+0000 mgr.y (mgr.14373) 184 : cluster 0 pgmap v163: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail; 323 KiB/s rd, 34 KiB/s wr, 424 op/s 2024-04-13T07:34:19.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:18 smithi203 bash[23480]: cluster 2024-04-13T07:34:17.586972+0000 mon.a (mon.0) 925 : cluster 0 osdmap e82: 8 total, 8 up, 8 in 2024-04-13T07:34:19.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:18 smithi203 bash[23480]: cluster 2024-04-13T07:34:17.901508+0000 mon.a (mon.0) 926 : cluster 0 osdmap e83: 8 total, 8 up, 8 in 2024-04-13T07:34:19.919 INFO:tasks.workunit.client.0.smithi089.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-04-13T07:34:19.932 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd snap purge testimg_import 2024-04-13T07:34:20.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:19 smithi089 bash[31522]: cluster 2024-04-13T07:34:18.904137+0000 mon.a (mon.0) 927 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-04-13T07:34:20.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:19 smithi089 bash[31522]: cluster 2024-04-13T07:34:18.940409+0000 mgr.y (mgr.14373) 185 : cluster 0 pgmap v167: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:20.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:19 smithi089 bash[21353]: cluster 2024-04-13T07:34:18.904137+0000 mon.a (mon.0) 927 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-04-13T07:34:20.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:19 smithi089 bash[21353]: cluster 2024-04-13T07:34:18.940409+0000 mgr.y (mgr.14373) 185 : cluster 0 pgmap v167: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:20.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:19 smithi203 bash[23480]: cluster 2024-04-13T07:34:18.904137+0000 mon.a (mon.0) 927 : cluster 0 osdmap e84: 8 total, 8 up, 8 in 2024-04-13T07:34:20.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:19 smithi203 bash[23480]: cluster 2024-04-13T07:34:18.940409+0000 mgr.y (mgr.14373) 185 : cluster 0 pgmap v167: 73 pgs: 73 active+clean; 689 KiB data, 231 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (3m) 25s ago 4m 13.8M - 0.25.0 c8568f914cd2 d39c47986efe 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi203 *:3000 running (3m) 24s ago 3m 78.5M - 9.4.7 954c08fa6188 4496ec520de5 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.pccnvq smithi089 running (2m) 25s ago 2m 11.8M - 18.2.2-1217-g7d36322c 6ecd1226aa2f 07167bfc6e5f 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi203.sjspdy smithi203 running (2m) 24s ago 2m 13.3M - 18.2.2-1217-g7d36322c 6ecd1226aa2f e100a52f8447 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi203 *:8443,8765 running (12m) 24s ago 12m 435M - 18.2.2-1217-g7d36322c 6ecd1226aa2f 827ca9b243c3 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:9283,8765 running (14m) 25s ago 14m 486M - 18.2.2-1217-g7d36322c 6ecd1226aa2f 8c4e123d6352 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (14m) 25s ago 15m 48.3M 2048M 18.2.2-1217-g7d36322c 6ecd1226aa2f 835fb959f8c6 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi203 running (12m) 24s ago 12m 37.6M 2048M 18.2.2-1217-g7d36322c 6ecd1226aa2f e475d5b92fda 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (12m) 25s ago 12m 36.7M 2048M 18.2.2-1217-g7d36322c 6ecd1226aa2f 2b26393a116d 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (4m) 25s ago 4m 8252k - 1.5.0 0da6a335fe13 04462903d931 2024-04-13T07:34:21.076 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi203 *:9100 running (4m) 24s ago 4m 8563k - 1.5.0 0da6a335fe13 a79084a2b37c 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (11m) 25s ago 11m 53.5M 4060M 18.2.2-1217-g7d36322c 6ecd1226aa2f 599c6c3ff3d6 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (10m) 25s ago 10m 44.4M 4060M 18.2.2-1217-g7d36322c 6ecd1226aa2f 58fb3beb85cc 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (10m) 25s ago 10m 43.4M 4060M 18.2.2-1217-g7d36322c 6ecd1226aa2f e020a590fd83 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (9m) 25s ago 9m 42.3M 4060M 18.2.2-1217-g7d36322c 6ecd1226aa2f 1c6772886276 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi203 running (8m) 24s ago 8m 51.1M 4060M 18.2.2-1217-g7d36322c 6ecd1226aa2f 28af48f1a163 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi203 running (7m) 24s ago 7m 42.2M 4060M 18.2.2-1217-g7d36322c 6ecd1226aa2f afe79fa4d7d0 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi203 running (6m) 24s ago 6m 42.2M 4060M 18.2.2-1217-g7d36322c 6ecd1226aa2f 8913a43d05b8 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi203 running (5m) 24s ago 5m 49.9M 4060M 18.2.2-1217-g7d36322c 6ecd1226aa2f 01ed33979f80 2024-04-13T07:34:21.077 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi203 *:9095 running (3m) 24s ago 4m 40.9M - 2.43.0 a07b618ecd1d 7c1a6eec2ad0 2024-04-13T07:34:21.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:20 smithi203 bash[23480]: cluster 2024-04-13T07:34:19.917599+0000 mon.a (mon.0) 928 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-04-13T07:34:21.286 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:20 smithi089 bash[21353]: cluster 2024-04-13T07:34:19.917599+0000 mon.a (mon.0) 928 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-04-13T07:34:21.286 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:20 smithi089 bash[31522]: cluster 2024-04-13T07:34:19.917599+0000 mon.a (mon.0) 928 : cluster 0 osdmap e85: 8 total, 8 up, 8 in 2024-04-13T07:34:21.939 INFO:tasks.workunit.client.0.smithi089.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2024-04-13T07:34:21.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg 2024-04-13T07:34:21.975 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'echo wait for minority of mons to upgrade' 2024-04-13T07:34:22.094 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:22.097 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd remove testimg_import 2024-04-13T07:34:22.258 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:22.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:21 smithi203 bash[23480]: cluster 2024-04-13T07:34:20.914606+0000 mon.a (mon.0) 929 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-04-13T07:34:22.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:21 smithi203 bash[23480]: cluster 2024-04-13T07:34:20.940756+0000 mgr.y (mgr.14373) 186 : cluster 0 pgmap v170: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 151 KiB/s rd, 14 KiB/s wr, 200 op/s 2024-04-13T07:34:22.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:21 smithi203 bash[23480]: audit 2024-04-13T07:34:21.068206+0000 mgr.y (mgr.14373) 187 : audit 0 from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:34:22.264 INFO:tasks.workunit.client.0.smithi089.stderr:+ tiered=0 2024-04-13T07:34:22.264 INFO:tasks.workunit.client.0.smithi089.stderr:+ ceph osd dump 2024-04-13T07:34:22.264 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep ^pool 2024-04-13T07:34:22.264 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep 'rbd' 2024-04-13T07:34:22.264 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep tier 2024-04-13T07:34:22.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:21 smithi089 bash[31522]: cluster 2024-04-13T07:34:20.914606+0000 mon.a (mon.0) 929 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-04-13T07:34:22.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:21 smithi089 bash[31522]: cluster 2024-04-13T07:34:20.940756+0000 mgr.y (mgr.14373) 186 : cluster 0 pgmap v170: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 151 KiB/s rd, 14 KiB/s wr, 200 op/s 2024-04-13T07:34:22.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:21 smithi089 bash[31522]: audit 2024-04-13T07:34:21.068206+0000 mgr.y (mgr.14373) 187 : audit 0 from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:34:22.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:21 smithi089 bash[21353]: cluster 2024-04-13T07:34:20.914606+0000 mon.a (mon.0) 929 : cluster 0 osdmap e86: 8 total, 8 up, 8 in 2024-04-13T07:34:22.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:21 smithi089 bash[21353]: cluster 2024-04-13T07:34:20.940756+0000 mgr.y (mgr.14373) 186 : cluster 0 pgmap v170: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 151 KiB/s rd, 14 KiB/s wr, 200 op/s 2024-04-13T07:34:22.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:21 smithi089 bash[21353]: audit 2024-04-13T07:34:21.068206+0000 mgr.y (mgr.14373) 187 : audit 0 from='client.15099 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:34:22.687 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_69261/sparse1 2024-04-13T07:34:22.693 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records in 2024-04-13T07:34:22.693 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records out 2024-04-13T07:34:22.693 INFO:tasks.workunit.client.0.smithi089.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00492087 s, 213 MB/s 2024-04-13T07:34:22.693 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_69261/sparse2 2024-04-13T07:34:22.698 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records in 2024-04-13T07:34:22.698 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records out 2024-04-13T07:34:22.698 INFO:tasks.workunit.client.0.smithi089.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00479542 s, 219 MB/s 2024-04-13T07:34:22.698 INFO:tasks.workunit.client.0.smithi089.stderr:+ truncate /tmp/rbd_import_export_69261/sparse2 -s 2M 2024-04-13T07:34:22.708 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse1 2024-04-13T07:34:22.762 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 0% complete...failed. 2024-04-13T07:34:22.762 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: delete error: (2) No such file or directory 2024-04-13T07:34:22.765 INFO:tasks.workunit.client.0.smithi089.stderr:+ true 2024-04-13T07:34:22.766 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_69261/sparse1 2024-04-13T07:34:22.844 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-04-13T07:34:22.844 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:22.844 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --order is deprecated, use --object-size 2024-04-13T07:34:22.848 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd ls -l 2024-04-13T07:34:22.848 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep sparse1 2024-04-13T07:34:22.848 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep -Ei (2 MiB|2048k) 2024-04-13T07:34:22.896 INFO:tasks.workunit.client.0.smithi089.stdout:sparse1 2 MiB 2 2024-04-13T07:34:22.897 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects sparse1 2024-04-13T07:34:22.897 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse1 2024-04-13T07:34:22.897 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse1 2024-04-13T07:34:22.897 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep block_name_prefix 2024-04-13T07:34:22.897 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $NF;} 2024-04-13T07:34:22.956 INFO:tasks.workunit.client.0.smithi089.stderr:+ prefix=rbd_data.3b19cc1703b6 2024-04-13T07:34:22.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep rbd_data.3b19cc1703b6 2024-04-13T07:34:22.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ get_image_data_pool sparse1 2024-04-13T07:34:22.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse1+ 2024-04-13T07:34:22.957 INFO:tasks.workunit.client.0.smithi089.stderr:sed -e s/rbd_data.3b19cc1703b6\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-04-13T07:34:22.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ sort -u 2024-04-13T07:34:22.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse1 2024-04-13T07:34:22.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep data_pool: 2024-04-13T07:34:22.957 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk -F: { print $NF } 2024-04-13T07:34:23.006 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool= 2024-04-13T07:34:23.006 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ -z ] 2024-04-13T07:34:23.006 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool=rbd 2024-04-13T07:34:23.007 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo rbd 2024-04-13T07:34:23.007 INFO:tasks.workunit.client.0.smithi089.stderr:+ rados ls -p rbd 2024-04-13T07:34:23.050 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects=1 2024-04-13T07:34:23.050 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo 1 2024-04-13T07:34:23.050 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2024-04-13T07:34:23.050 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export sparse1 /tmp/rbd_import_export_69261/sparse1.out 2024-04-13T07:34:23.094 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:23.098 INFO:tasks.workunit.client.0.smithi089.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69261/sparse1 /tmp/rbd_import_export_69261/sparse1.out 2024-04-13T07:34:23.098 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp -l /tmp/rbd_import_export_69261/sparse1 /tmp/rbd_import_export_69261/sparse1.out 2024-04-13T07:34:23.100 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse1 --format %b 2024-04-13T07:34:23.100 INFO:tasks.workunit.client.0.smithi089.stderr:+ origsize=2048 2024-04-13T07:34:23.101 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse1.out --format %b 2024-04-13T07:34:23.101 INFO:tasks.workunit.client.0.smithi089.stderr:+ exportsize=2048 2024-04-13T07:34:23.101 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:23.102 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:23.102 INFO:tasks.workunit.client.0.smithi089.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-13T07:34:23.102 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/sparse1.out 2024-04-13T07:34:23.102 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse1 2024-04-13T07:34:23.207 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-04-13T07:34:23.211 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse2 2024-04-13T07:34:23.257 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 0% complete...failed. 2024-04-13T07:34:23.257 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: delete error: (2) No such file or directory 2024-04-13T07:34:23.261 INFO:tasks.workunit.client.0.smithi089.stderr:+ true 2024-04-13T07:34:23.261 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_69261/sparse2 2024-04-13T07:34:23.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:22 smithi203 bash[23480]: cluster 2024-04-13T07:34:21.936408+0000 mon.a (mon.0) 930 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-04-13T07:34:23.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:22 smithi203 bash[23480]: audit 2024-04-13T07:34:22.566818+0000 mon.a (mon.0) 931 : audit 0 from='client.? 172.21.15.89:0/1285003333' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-13T07:34:23.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:22 smithi203 bash[23480]: audit 2024-04-13T07:34:22.676069+0000 mon.a (mon.0) 932 : audit 0 from='client.? 172.21.15.89:0/1285003333' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-13T07:34:23.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:22 smithi203 bash[23480]: cluster 2024-04-13T07:34:22.899351+0000 mon.a (mon.0) 933 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-04-13T07:34:23.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:22 smithi089 bash[31522]: cluster 2024-04-13T07:34:21.936408+0000 mon.a (mon.0) 930 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-04-13T07:34:23.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:22 smithi089 bash[31522]: audit 2024-04-13T07:34:22.566818+0000 mon.a (mon.0) 931 : audit 0 from='client.? 172.21.15.89:0/1285003333' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-13T07:34:23.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:22 smithi089 bash[31522]: audit 2024-04-13T07:34:22.676069+0000 mon.a (mon.0) 932 : audit 0 from='client.? 172.21.15.89:0/1285003333' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-13T07:34:23.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:22 smithi089 bash[31522]: cluster 2024-04-13T07:34:22.899351+0000 mon.a (mon.0) 933 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-04-13T07:34:23.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:22 smithi089 bash[21353]: cluster 2024-04-13T07:34:21.936408+0000 mon.a (mon.0) 930 : cluster 0 osdmap e87: 8 total, 8 up, 8 in 2024-04-13T07:34:23.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:22 smithi089 bash[21353]: audit 2024-04-13T07:34:22.566818+0000 mon.a (mon.0) 931 : audit 0 from='client.? 172.21.15.89:0/1285003333' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-13T07:34:23.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:22 smithi089 bash[21353]: audit 2024-04-13T07:34:22.676069+0000 mon.a (mon.0) 932 : audit 0 from='client.? 172.21.15.89:0/1285003333' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2024-04-13T07:34:23.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:22 smithi089 bash[21353]: cluster 2024-04-13T07:34:22.899351+0000 mon.a (mon.0) 933 : cluster 0 osdmap e88: 8 total, 8 up, 8 in 2024-04-13T07:34:23.333 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2024-04-13T07:34:23.334 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:23.334 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --order is deprecated, use --object-size 2024-04-13T07:34:23.337 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd ls -l 2024-04-13T07:34:23.337 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep sparse2 2024-04-13T07:34:23.337 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep -Ei (2 MiB|2048k) 2024-04-13T07:34:23.378 INFO:tasks.workunit.client.0.smithi089.stdout:sparse2 2 MiB 2 2024-04-13T07:34:23.379 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects sparse2 2024-04-13T07:34:23.379 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse2 2024-04-13T07:34:23.379 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse2 2024-04-13T07:34:23.379 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep block_name_prefix 2024-04-13T07:34:23.379 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $NF;} 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:+ prefix=rbd_data.3b4c22790b63 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep rbd_data.3b4c22790b63 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:+ get_image_data_pool sparse2 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse2 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:+ sed -e s/rbd_data.3b4c22790b63\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:+ sort -u 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse2 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:+ + awk -F: { print $NF } 2024-04-13T07:34:23.422 INFO:tasks.workunit.client.0.smithi089.stderr:grep data_pool: 2024-04-13T07:34:23.460 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool= 2024-04-13T07:34:23.461 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ -z ] 2024-04-13T07:34:23.461 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool=rbd 2024-04-13T07:34:23.461 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo rbd 2024-04-13T07:34:23.461 INFO:tasks.workunit.client.0.smithi089.stderr:+ rados ls -p rbd 2024-04-13T07:34:23.501 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects=0 2024-04-13T07:34:23.502 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo 0 2024-04-13T07:34:23.502 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2024-04-13T07:34:23.502 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export sparse2 /tmp/rbd_import_export_69261/sparse2.out 2024-04-13T07:34:23.546 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:23.550 INFO:tasks.workunit.client.0.smithi089.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69261/sparse2 /tmp/rbd_import_export_69261/sparse2.out 2024-04-13T07:34:23.550 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp -l /tmp/rbd_import_export_69261/sparse2 /tmp/rbd_import_export_69261/sparse2.out 2024-04-13T07:34:23.552 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse2 --format %b 2024-04-13T07:34:23.552 INFO:tasks.workunit.client.0.smithi089.stderr:+ origsize=2048 2024-04-13T07:34:23.553 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse2.out --format %b 2024-04-13T07:34:23.553 INFO:tasks.workunit.client.0.smithi089.stderr:+ exportsize=2048 2024-04-13T07:34:23.554 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:23.554 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:23.554 INFO:tasks.workunit.client.0.smithi089.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-13T07:34:23.554 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/sparse2.out 2024-04-13T07:34:23.554 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse2 2024-04-13T07:34:23.661 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2024-04-13T07:34:23.665 INFO:tasks.workunit.client.0.smithi089.stderr:+ truncate /tmp/rbd_import_export_69261/sparse1 -s 10M 2024-04-13T07:34:23.665 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format --order 20 - sparse1 2024-04-13T07:34:23.775 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:23.776 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:23.776 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --order is deprecated, use --object-size 2024-04-13T07:34:23.779 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd ls -l 2024-04-13T07:34:23.779 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep sparse1 2024-04-13T07:34:23.779 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep -Ei (10 MiB|10240k) 2024-04-13T07:34:23.822 INFO:tasks.workunit.client.0.smithi089.stdout:sparse1 10 MiB 2 2024-04-13T07:34:23.823 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects sparse1 2024-04-13T07:34:23.823 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse1 2024-04-13T07:34:23.823 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse1 2024-04-13T07:34:23.823 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep block_name_prefix 2024-04-13T07:34:23.823 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $NF;} 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:+ prefix=rbd_data.3b796aaf9b02 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep rbd_data.3b796aaf9b02 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:+ + sed -e s/rbd_data.3b796aaf9b02\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:get_image_data_pool sparse1 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse1 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:+ sort -u 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse1 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep data_pool: 2024-04-13T07:34:23.865 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk -F: { print $NF } 2024-04-13T07:34:23.904 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool= 2024-04-13T07:34:23.904 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ -z ] 2024-04-13T07:34:23.905 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool=rbd 2024-04-13T07:34:23.905 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo rbd 2024-04-13T07:34:23.905 INFO:tasks.workunit.client.0.smithi089.stderr:+ rados ls -p rbd 2024-04-13T07:34:23.950 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects=1 2024-04-13T07:34:23.950 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo 1 2024-04-13T07:34:23.950 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2024-04-13T07:34:23.950 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export sparse1 /tmp/rbd_import_export_69261/sparse1.out 2024-04-13T07:34:23.996 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:24.000 INFO:tasks.workunit.client.0.smithi089.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69261/sparse1 /tmp/rbd_import_export_69261/sparse1.out 2024-04-13T07:34:24.000 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp -l /tmp/rbd_import_export_69261/sparse1 /tmp/rbd_import_export_69261/sparse1.out 2024-04-13T07:34:24.007 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse1 --format %b 2024-04-13T07:34:24.008 INFO:tasks.workunit.client.0.smithi089.stderr:+ origsize=2048 2024-04-13T07:34:24.008 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse1.out --format %b 2024-04-13T07:34:24.009 INFO:tasks.workunit.client.0.smithi089.stderr:+ exportsize=2048 2024-04-13T07:34:24.009 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:24.009 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:24.009 INFO:tasks.workunit.client.0.smithi089.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-13T07:34:24.009 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/sparse1.out 2024-04-13T07:34:24.010 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse1 2024-04-13T07:34:24.133 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-04-13T07:34:24.137 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_69261/sparse2 oflag=append conv=notrunc 2024-04-13T07:34:24.146 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records in 2024-04-13T07:34:24.146 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records out 2024-04-13T07:34:24.147 INFO:tasks.workunit.client.0.smithi089.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.00828044 s, 253 MB/s 2024-04-13T07:34:24.147 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format --order 20 - sparse2 2024-04-13T07:34:24.258 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:24.259 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:24.259 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --order is deprecated, use --object-size 2024-04-13T07:34:24.263 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd ls -l 2024-04-13T07:34:24.263 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep sparse2 2024-04-13T07:34:24.263 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep -Ei (4 MiB|4096k) 2024-04-13T07:34:24.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:23 smithi203 bash[23480]: cluster 2024-04-13T07:34:22.941064+0000 mgr.y (mgr.14373) 188 : cluster 0 pgmap v173: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2024-04-13T07:34:24.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:23 smithi089 bash[21353]: cluster 2024-04-13T07:34:22.941064+0000 mgr.y (mgr.14373) 188 : cluster 0 pgmap v173: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2024-04-13T07:34:24.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:23 smithi089 bash[31522]: cluster 2024-04-13T07:34:22.941064+0000 mgr.y (mgr.14373) 188 : cluster 0 pgmap v173: 73 pgs: 73 active+clean; 799 KiB data, 232 MiB used, 715 GiB / 715 GiB avail; 127 KiB/s rd, 12 KiB/s wr, 168 op/s 2024-04-13T07:34:24.308 INFO:tasks.workunit.client.0.smithi089.stdout:sparse2 4 MiB 2 2024-04-13T07:34:24.309 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects sparse2 2024-04-13T07:34:24.309 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse2 2024-04-13T07:34:24.309 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse2 2024-04-13T07:34:24.309 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep block_name_prefix 2024-04-13T07:34:24.310 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $NF;} 2024-04-13T07:34:24.356 INFO:tasks.workunit.client.0.smithi089.stderr:+ prefix=rbd_data.3ba6f4b51c0e 2024-04-13T07:34:24.356 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep rbd_data.3ba6f4b51c0e 2024-04-13T07:34:24.357 INFO:tasks.workunit.client.0.smithi089.stderr:+ get_image_data_pool sparse2 2024-04-13T07:34:24.357 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse2 2024-04-13T07:34:24.357 INFO:tasks.workunit.client.0.smithi089.stderr:+ sed -e s/rbd_data.3ba6f4b51c0e\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-04-13T07:34:24.357 INFO:tasks.workunit.client.0.smithi089.stderr:+ sort -u 2024-04-13T07:34:24.357 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse2 2024-04-13T07:34:24.357 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep data_pool: 2024-04-13T07:34:24.357 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk -F: { print $NF } 2024-04-13T07:34:24.400 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool= 2024-04-13T07:34:24.400 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ -z ] 2024-04-13T07:34:24.400 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool=rbd 2024-04-13T07:34:24.400 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo rbd 2024-04-13T07:34:24.400 INFO:tasks.workunit.client.0.smithi089.stderr:+ rados ls -p rbd 2024-04-13T07:34:24.439 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects=0 2024-04-13T07:34:24.439 INFO:tasks.workunit.client.0.smithi089.stderr:2 2024-04-13T07:34:24.440 INFO:tasks.workunit.client.0.smithi089.stderr:3 2024-04-13T07:34:24.440 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo 0 2 3 2024-04-13T07:34:24.440 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ 0 -eq 1 -o 0 2 3 = 0 2 3 ] 2024-04-13T07:34:24.440 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export sparse2 /tmp/rbd_import_export_69261/sparse2.out 2024-04-13T07:34:24.489 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2024-04-13T07:34:24.494 INFO:tasks.workunit.client.0.smithi089.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_69261/sparse2 /tmp/rbd_import_export_69261/sparse2.out 2024-04-13T07:34:24.494 INFO:tasks.workunit.client.0.smithi089.stderr:+ cmp -l /tmp/rbd_import_export_69261/sparse2 /tmp/rbd_import_export_69261/sparse2.out 2024-04-13T07:34:24.497 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse2 --format %b 2024-04-13T07:34:24.498 INFO:tasks.workunit.client.0.smithi089.stderr:+ origsize=6144 2024-04-13T07:34:24.498 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse2.out --format %b 2024-04-13T07:34:24.499 INFO:tasks.workunit.client.0.smithi089.stderr:+ exportsize=6144 2024-04-13T07:34:24.499 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:24.499 INFO:tasks.workunit.client.0.smithi089.stderr:+ difference=0 2024-04-13T07:34:24.499 INFO:tasks.workunit.client.0.smithi089.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2024-04-13T07:34:24.499 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/sparse2.out 2024-04-13T07:34:24.499 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse2 2024-04-13T07:34:24.630 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2024-04-13T07:34:24.633 INFO:tasks.workunit.client.0.smithi089.stdout:partially-sparse file imports to partially-sparse image 2024-04-13T07:34:24.634 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo partially-sparse file imports to partially-sparse image 2024-04-13T07:34:24.634 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_69261/sparse1 sparse 2024-04-13T07:34:24.725 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2024-04-13T07:34:24.725 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:24.725 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --order is deprecated, use --object-size 2024-04-13T07:34:24.730 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects sparse 2024-04-13T07:34:24.730 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse 2024-04-13T07:34:24.730 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse 2024-04-13T07:34:24.730 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep block_name_prefix 2024-04-13T07:34:24.730 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $NF;} 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:+ prefix=rbd_data.3bd386d68eb3 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep rbd_data.3bd386d68eb3 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:+ + get_image_data_pool sparse 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:sed -e s/rbd_data.3bd386d68eb3\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:+ sort -u 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep data_pool: 2024-04-13T07:34:24.775 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk -F: { print $NF } 2024-04-13T07:34:24.819 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool= 2024-04-13T07:34:24.819 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ -z ] 2024-04-13T07:34:24.819 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool=rbd 2024-04-13T07:34:24.819 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo rbd 2024-04-13T07:34:24.819 INFO:tasks.workunit.client.0.smithi089.stderr:+ rados ls -p rbd 2024-04-13T07:34:24.858 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects=1 2024-04-13T07:34:24.858 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo 1 2024-04-13T07:34:24.858 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ 0 -eq 1 -o 1 = 1 ] 2024-04-13T07:34:24.859 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse 2024-04-13T07:34:25.012 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2024-04-13T07:34:25.016 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo zeros import through stdin to sparse image 2024-04-13T07:34:25.016 INFO:tasks.workunit.client.0.smithi089.stdout:zeros import through stdin to sparse image 2024-04-13T07:34:25.017 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/dev/zero bs=1M count=4 2024-04-13T07:34:25.017 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd import --new-format - sparse 2024-04-13T07:34:25.076 INFO:tasks.workunit.client.0.smithi089.stderr:4+0 records in 2024-04-13T07:34:25.076 INFO:tasks.workunit.client.0.smithi089.stderr:4+0 records out 2024-04-13T07:34:25.076 INFO:tasks.workunit.client.0.smithi089.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0590677 s, 71.0 MB/s 2024-04-13T07:34:25.094 INFO:tasks.workunit.client.0.smithi089.stderr: Importing image: 100% complete...done. 2024-04-13T07:34:25.094 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:25.098 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects sparse 2024-04-13T07:34:25.098 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse 2024-04-13T07:34:25.098 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse 2024-04-13T07:34:25.098 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep block_name_prefix 2024-04-13T07:34:25.098 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $NF;} 2024-04-13T07:34:25.144 INFO:tasks.workunit.client.0.smithi089.stderr:+ prefix=rbd_data.3bf436cba91b 2024-04-13T07:34:25.145 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep rbd_data.3bf436cba91b 2024-04-13T07:34:25.145 INFO:tasks.workunit.client.0.smithi089.stderr:+ get_image_data_pool sparse 2024-04-13T07:34:25.146 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse 2024-04-13T07:34:25.146 INFO:tasks.workunit.client.0.smithi089.stderr:+ sed -e s/rbd_data.3bf436cba91b\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-04-13T07:34:25.146 INFO:tasks.workunit.client.0.smithi089.stderr:+ sort -u 2024-04-13T07:34:25.146 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse 2024-04-13T07:34:25.146 INFO:tasks.workunit.client.0.smithi089.stderr:+ + grep data_pool: 2024-04-13T07:34:25.146 INFO:tasks.workunit.client.0.smithi089.stderr:awk -F: { print $NF } 2024-04-13T07:34:25.190 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool= 2024-04-13T07:34:25.190 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ -z ] 2024-04-13T07:34:25.190 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool=rbd 2024-04-13T07:34:25.190 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo rbd 2024-04-13T07:34:25.190 INFO:tasks.workunit.client.0.smithi089.stderr:+ rados ls -p rbd 2024-04-13T07:34:25.230 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects= 2024-04-13T07:34:25.230 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo 2024-04-13T07:34:25.230 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ 0 -eq 1 -o = ] 2024-04-13T07:34:25.230 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse 2024-04-13T07:34:25.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:24 smithi203 bash[23480]: audit 2024-04-13T07:34:23.934499+0000 mon.a (mon.0) 934 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:25.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:24 smithi089 bash[21353]: audit 2024-04-13T07:34:23.934499+0000 mon.a (mon.0) 934 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:25.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:24 smithi089 bash[31522]: audit 2024-04-13T07:34:23.934499+0000 mon.a (mon.0) 934 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:25.361 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:25.364 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo zeros export to sparse file 2024-04-13T07:34:25.365 INFO:tasks.workunit.client.0.smithi089.stdout:zeros export to sparse file 2024-04-13T07:34:25.365 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd create --new-format sparse --size 4 2024-04-13T07:34:25.413 INFO:tasks.workunit.client.0.smithi089.stderr:rbd: --new-format is deprecated, use --image-format 2 2024-04-13T07:34:25.416 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse 2024-04-13T07:34:25.416 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep block_name_prefix 2024-04-13T07:34:25.416 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $NF;} 2024-04-13T07:34:25.462 INFO:tasks.workunit.client.0.smithi089.stderr:+ prefix=rbd_data.3c1562d56995 2024-04-13T07:34:25.462 INFO:tasks.workunit.client.0.smithi089.stderr:+ dd if=/dev/zero bs=4M count=1 2024-04-13T07:34:25.462 INFO:tasks.workunit.client.0.smithi089.stderr:+ get_image_data_pool sparse 2024-04-13T07:34:25.463 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse 2024-04-13T07:34:25.463 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse 2024-04-13T07:34:25.463 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep data_pool: 2024-04-13T07:34:25.463 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk -F: { print $NF } 2024-04-13T07:34:25.507 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool= 2024-04-13T07:34:25.507 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ -z ] 2024-04-13T07:34:25.507 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool=rbd 2024-04-13T07:34:25.507 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo rbd 2024-04-13T07:34:25.507 INFO:tasks.workunit.client.0.smithi089.stderr:+ rados -p rbd put rbd_data.3c1562d56995.000000000000 - 2024-04-13T07:34:25.539 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records in 2024-04-13T07:34:25.539 INFO:tasks.workunit.client.0.smithi089.stderr:1+0 records out 2024-04-13T07:34:25.539 INFO:tasks.workunit.client.0.smithi089.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0768264 s, 54.6 MB/s 2024-04-13T07:34:25.567 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects sparse 2024-04-13T07:34:25.567 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse 2024-04-13T07:34:25.568 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info sparse 2024-04-13T07:34:25.568 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep block_name_prefix 2024-04-13T07:34:25.568 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk {print $NF;} 2024-04-13T07:34:25.616 INFO:tasks.workunit.client.0.smithi089.stderr:+ prefix=rbd_data.3c1562d56995 2024-04-13T07:34:25.616 INFO:tasks.workunit.client.0.smithi089.stderr:+ grep rbd_data.3c1562d56995 2024-04-13T07:34:25.616 INFO:tasks.workunit.client.0.smithi089.stderr:+ + get_image_data_pool sparse 2024-04-13T07:34:25.616 INFO:tasks.workunit.client.0.smithi089.stderr:+ image=sparse 2024-04-13T07:34:25.616 INFO:tasks.workunit.client.0.smithi089.stderr:sed -e s/rbd_data.3c1562d56995\.// -e s/^0*\([0-9a-f]\)/\1/ 2024-04-13T07:34:25.616 INFO:tasks.workunit.client.0.smithi089.stderr:+ sort -u 2024-04-13T07:34:25.617 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd info+ grep data_pool: 2024-04-13T07:34:25.617 INFO:tasks.workunit.client.0.smithi089.stderr: sparse 2024-04-13T07:34:25.617 INFO:tasks.workunit.client.0.smithi089.stderr:+ awk -F: { print $NF } 2024-04-13T07:34:25.663 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool= 2024-04-13T07:34:25.664 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ -z ] 2024-04-13T07:34:25.664 INFO:tasks.workunit.client.0.smithi089.stderr:+ data_pool=rbd 2024-04-13T07:34:25.664 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo rbd 2024-04-13T07:34:25.664 INFO:tasks.workunit.client.0.smithi089.stderr:+ rados ls -p rbd 2024-04-13T07:34:25.704 INFO:tasks.workunit.client.0.smithi089.stderr:+ objects=0 2024-04-13T07:34:25.704 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo 0 2024-04-13T07:34:25.704 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ 0 -eq 1 -o 0 = 0 ] 2024-04-13T07:34:25.704 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/sparse 2024-04-13T07:34:25.704 INFO:tasks.workunit.client.0.smithi089.stderr:rm: cannot remove '/tmp/rbd_import_export_69261/sparse': No such file or directory 2024-04-13T07:34:25.704 INFO:tasks.workunit.client.0.smithi089.stderr:+ true 2024-04-13T07:34:25.704 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd export sparse /tmp/rbd_import_export_69261/sparse 2024-04-13T07:34:25.748 INFO:tasks.workunit.client.0.smithi089.stderr: Exporting image: 100% complete...done. 2024-04-13T07:34:25.752 INFO:tasks.workunit.client.0.smithi089.stderr:+ stat /tmp/rbd_import_export_69261/sparse --format=%b 2024-04-13T07:34:25.752 INFO:tasks.workunit.client.0.smithi089.stderr:+ [ 0 = 0 ] 2024-04-13T07:34:25.752 INFO:tasks.workunit.client.0.smithi089.stderr:+ rbd rm sparse 2024-04-13T07:34:25.897 INFO:tasks.workunit.client.0.smithi089.stderr: Removing image: 100% complete...done. 2024-04-13T07:34:25.900 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm /tmp/rbd_import_export_69261/sparse /tmp/rbd_import_export_69261/sparse1 /tmp/rbd_import_export_69261/sparse2 /tmp/rbd_import_export_69261/sparse3 2024-04-13T07:34:25.902 INFO:tasks.workunit.client.0.smithi089.stderr:rm: cannot remove '/tmp/rbd_import_export_69261/sparse3': No such file or directory 2024-04-13T07:34:25.902 INFO:tasks.workunit.client.0.smithi089.stderr:+ true 2024-04-13T07:34:25.902 INFO:tasks.workunit.client.0.smithi089.stdout:OK 2024-04-13T07:34:25.903 INFO:tasks.workunit.client.0.smithi089.stderr:+ echo OK 2024-04-13T07:34:25.903 INFO:tasks.workunit.client.0.smithi089.stderr:+ rm -rf /tmp/rbd_import_export_69261 2024-04-13T07:34:25.904 INFO:teuthology.orchestra.run:Running command with timeout 3600 2024-04-13T07:34:25.904 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2024-04-13T07:34:25.913 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2024-04-13T07:34:25.913 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2024-04-13T07:34:26.167 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:25 smithi089 bash[21353]: cluster 2024-04-13T07:34:24.941605+0000 mgr.y (mgr.14373) 189 : cluster 0 pgmap v174: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 544 op/s 2024-04-13T07:34:26.168 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:25 smithi089 bash[31522]: cluster 2024-04-13T07:34:24.941605+0000 mgr.y (mgr.14373) 189 : cluster 0 pgmap v174: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 544 op/s 2024-04-13T07:34:26.180 DEBUG:teuthology.parallel:result is None 2024-04-13T07:34:26.181 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2024-04-13T07:34:26.188 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:34:26 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:34:26] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.43.0" 2024-04-13T07:34:26.191 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2024-04-13T07:34:26.191 DEBUG:teuthology.orchestra.run.smithi089:> rmdir -- /home/ubuntu/cephtest/mnt.0 2024-04-13T07:34:26.238 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2024-04-13T07:34:26.238 INFO:teuthology.task.sequential:In sequential, running task print... 2024-04-13T07:34:26.239 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2024-04-13T07:34:26.239 DEBUG:teuthology.parallel:result is None 2024-04-13T07:34:26.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:25 smithi203 bash[23480]: cluster 2024-04-13T07:34:24.941605+0000 mgr.y (mgr.14373) 189 : cluster 0 pgmap v174: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.6 MiB/s rd, 1.4 MiB/s wr, 544 op/s 2024-04-13T07:34:26.668 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:34:28.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:27 smithi203 bash[23480]: cluster 2024-04-13T07:34:26.942206+0000 mgr.y (mgr.14373) 190 : cluster 0 pgmap v175: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 342 op/s 2024-04-13T07:34:28.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:27 smithi203 bash[23480]: cluster 2024-04-13T07:34:27.907253+0000 mon.a (mon.0) 935 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-04-13T07:34:28.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:27 smithi089 bash[21353]: cluster 2024-04-13T07:34:26.942206+0000 mgr.y (mgr.14373) 190 : cluster 0 pgmap v175: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 342 op/s 2024-04-13T07:34:28.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:27 smithi089 bash[21353]: cluster 2024-04-13T07:34:27.907253+0000 mon.a (mon.0) 935 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-04-13T07:34:28.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:27 smithi089 bash[31522]: cluster 2024-04-13T07:34:26.942206+0000 mgr.y (mgr.14373) 190 : cluster 0 pgmap v175: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.2 MiB/s rd, 1.2 MiB/s wr, 342 op/s 2024-04-13T07:34:28.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:27 smithi089 bash[31522]: cluster 2024-04-13T07:34:27.907253+0000 mon.a (mon.0) 935 : cluster 0 osdmap e89: 8 total, 8 up, 8 in 2024-04-13T07:34:30.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:29 smithi203 bash[23480]: cluster 2024-04-13T07:34:28.942690+0000 mgr.y (mgr.14373) 191 : cluster 0 pgmap v177: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.0 MiB/s wr, 294 op/s 2024-04-13T07:34:30.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:29 smithi089 bash[31522]: cluster 2024-04-13T07:34:28.942690+0000 mgr.y (mgr.14373) 191 : cluster 0 pgmap v177: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.0 MiB/s wr, 294 op/s 2024-04-13T07:34:30.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:29 smithi089 bash[21353]: cluster 2024-04-13T07:34:28.942690+0000 mgr.y (mgr.14373) 191 : cluster 0 pgmap v177: 73 pgs: 73 active+clean; 1.6 MiB data, 247 MiB used, 715 GiB / 715 GiB avail; 1.1 MiB/s rd, 1.0 MiB/s wr, 294 op/s 2024-04-13T07:34:30.644 INFO:teuthology.orchestra.run.smithi089.stdout:wait for minority of mons to upgrade 2024-04-13T07:34:31.407 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2024-04-13T07:34:32.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:31 smithi203 bash[23480]: cluster 2024-04-13T07:34:30.943639+0000 mgr.y (mgr.14373) 192 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1015 KiB/s rd, 1.4 MiB/s wr, 337 op/s 2024-04-13T07:34:32.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:31 smithi089 bash[21353]: cluster 2024-04-13T07:34:30.943639+0000 mgr.y (mgr.14373) 192 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1015 KiB/s rd, 1.4 MiB/s wr, 337 op/s 2024-04-13T07:34:32.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:31 smithi089 bash[31522]: cluster 2024-04-13T07:34:30.943639+0000 mgr.y (mgr.14373) 192 : cluster 0 pgmap v178: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 1015 KiB/s rd, 1.4 MiB/s wr, 337 op/s 2024-04-13T07:34:34.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:33 smithi203 bash[23480]: cluster 2024-04-13T07:34:32.944049+0000 mgr.y (mgr.14373) 193 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-04-13T07:34:34.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:33 smithi089 bash[31522]: cluster 2024-04-13T07:34:32.944049+0000 mgr.y (mgr.14373) 193 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-04-13T07:34:34.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:33 smithi089 bash[21353]: cluster 2024-04-13T07:34:32.944049+0000 mgr.y (mgr.14373) 193 : cluster 0 pgmap v179: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 817 KiB/s rd, 1.1 MiB/s wr, 271 op/s 2024-04-13T07:34:36.106 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:34:36.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:35 smithi203 bash[23480]: cluster 2024-04-13T07:34:34.944674+0000 mgr.y (mgr.14373) 194 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2024-04-13T07:34:36.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:34:36 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:34:36] "GET /metrics HTTP/1.1" 200 33821 "" "Prometheus/2.43.0" 2024-04-13T07:34:36.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:35 smithi089 bash[31522]: cluster 2024-04-13T07:34:34.944674+0000 mgr.y (mgr.14373) 194 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2024-04-13T07:34:36.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:35 smithi089 bash[21353]: cluster 2024-04-13T07:34:34.944674+0000 mgr.y (mgr.14373) 194 : cluster 0 pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2024-04-13T07:34:38.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:37 smithi203 bash[23480]: cluster 2024-04-13T07:34:36.945020+0000 mgr.y (mgr.14373) 195 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2024-04-13T07:34:38.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:37 smithi089 bash[21353]: cluster 2024-04-13T07:34:36.945020+0000 mgr.y (mgr.14373) 195 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2024-04-13T07:34:38.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:37 smithi089 bash[31522]: cluster 2024-04-13T07:34:36.945020+0000 mgr.y (mgr.14373) 195 : cluster 0 pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s rd, 412 KiB/s wr, 64 op/s 2024-04-13T07:34:39.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:38 smithi203 bash[23480]: audit 2024-04-13T07:34:38.935107+0000 mon.a (mon.0) 936 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:39.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:38 smithi089 bash[21353]: audit 2024-04-13T07:34:38.935107+0000 mon.a (mon.0) 936 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:39.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:38 smithi089 bash[31522]: audit 2024-04-13T07:34:38.935107+0000 mon.a (mon.0) 936 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:40.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:39 smithi203 bash[23480]: cluster 2024-04-13T07:34:38.945389+0000 mgr.y (mgr.14373) 196 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 373 KiB/s wr, 58 op/s 2024-04-13T07:34:40.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:39 smithi089 bash[31522]: cluster 2024-04-13T07:34:38.945389+0000 mgr.y (mgr.14373) 196 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 373 KiB/s wr, 58 op/s 2024-04-13T07:34:40.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:39 smithi089 bash[21353]: cluster 2024-04-13T07:34:38.945389+0000 mgr.y (mgr.14373) 196 : cluster 0 pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 44 KiB/s rd, 373 KiB/s wr, 58 op/s 2024-04-13T07:34:42.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:41 smithi203 bash[23480]: cluster 2024-04-13T07:34:40.946479+0000 mgr.y (mgr.14373) 197 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 343 KiB/s wr, 53 op/s 2024-04-13T07:34:42.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:41 smithi089 bash[31522]: cluster 2024-04-13T07:34:40.946479+0000 mgr.y (mgr.14373) 197 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 343 KiB/s wr, 53 op/s 2024-04-13T07:34:42.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:41 smithi089 bash[21353]: cluster 2024-04-13T07:34:40.946479+0000 mgr.y (mgr.14373) 197 : cluster 0 pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 40 KiB/s rd, 343 KiB/s wr, 53 op/s 2024-04-13T07:34:44.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:43 smithi203 bash[23480]: cluster 2024-04-13T07:34:42.947081+0000 mgr.y (mgr.14373) 198 : cluster 0 pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:44.297 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:43 smithi089 bash[31522]: cluster 2024-04-13T07:34:42.947081+0000 mgr.y (mgr.14373) 198 : cluster 0 pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:43 smithi089 bash[21353]: cluster 2024-04-13T07:34:42.947081+0000 mgr.y (mgr.14373) 198 : cluster 0 pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:46.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:45 smithi203 bash[23480]: cluster 2024-04-13T07:34:44.947955+0000 mgr.y (mgr.14373) 199 : cluster 0 pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:46.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:45 smithi089 bash[21353]: cluster 2024-04-13T07:34:44.947955+0000 mgr.y (mgr.14373) 199 : cluster 0 pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:46.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:34:46 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:34:46] "GET /metrics HTTP/1.1" 200 33832 "" "Prometheus/2.43.0" 2024-04-13T07:34:46.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:45 smithi089 bash[31522]: cluster 2024-04-13T07:34:44.947955+0000 mgr.y (mgr.14373) 199 : cluster 0 pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:48.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:47 smithi203 bash[23480]: cluster 2024-04-13T07:34:46.948584+0000 mgr.y (mgr.14373) 200 : cluster 0 pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:48.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:47 smithi089 bash[21353]: cluster 2024-04-13T07:34:46.948584+0000 mgr.y (mgr.14373) 200 : cluster 0 pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:48.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:47 smithi089 bash[31522]: cluster 2024-04-13T07:34:46.948584+0000 mgr.y (mgr.14373) 200 : cluster 0 pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:50 smithi203 bash[23480]: cluster 2024-04-13T07:34:48.949023+0000 mgr.y (mgr.14373) 201 : cluster 0 pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:50 smithi203 bash[23480]: audit 2024-04-13T07:34:49.692011+0000 mon.a (mon.0) 937 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:50 smithi203 bash[23480]: cephadm 2024-04-13T07:34:49.692504+0000 mgr.y (mgr.14373) 202 : cephadm 1 Upgrade: Target is version 19.0.0-2905-g5f22a497 (unknown) 2024-04-13T07:34:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:50 smithi203 bash[23480]: cephadm 2024-04-13T07:34:49.692587+0000 mgr.y (mgr.14373) 203 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de, digests ['quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de'] 2024-04-13T07:34:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:50 smithi203 bash[23480]: audit 2024-04-13T07:34:49.694017+0000 mon.a (mon.0) 938 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:34:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:50 smithi203 bash[23480]: audit 2024-04-13T07:34:49.699342+0000 mon.a (mon.0) 939 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:50 smithi203 bash[23480]: cephadm 2024-04-13T07:34:49.700140+0000 mgr.y (mgr.14373) 204 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-04-13T07:34:51.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[21353]: cluster 2024-04-13T07:34:48.949023+0000 mgr.y (mgr.14373) 201 : cluster 0 pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:51.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[21353]: audit 2024-04-13T07:34:49.692011+0000 mon.a (mon.0) 937 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:51.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[21353]: cephadm 2024-04-13T07:34:49.692504+0000 mgr.y (mgr.14373) 202 : cephadm 1 Upgrade: Target is version 19.0.0-2905-g5f22a497 (unknown) 2024-04-13T07:34:51.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[21353]: cephadm 2024-04-13T07:34:49.692587+0000 mgr.y (mgr.14373) 203 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de, digests ['quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de'] 2024-04-13T07:34:51.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[21353]: audit 2024-04-13T07:34:49.694017+0000 mon.a (mon.0) 938 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[21353]: audit 2024-04-13T07:34:49.699342+0000 mon.a (mon.0) 939 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[21353]: cephadm 2024-04-13T07:34:49.700140+0000 mgr.y (mgr.14373) 204 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[31522]: cluster 2024-04-13T07:34:48.949023+0000 mgr.y (mgr.14373) 201 : cluster 0 pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[31522]: audit 2024-04-13T07:34:49.692011+0000 mon.a (mon.0) 937 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[31522]: cephadm 2024-04-13T07:34:49.692504+0000 mgr.y (mgr.14373) 202 : cephadm 1 Upgrade: Target is version 19.0.0-2905-g5f22a497 (unknown) 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[31522]: cephadm 2024-04-13T07:34:49.692587+0000 mgr.y (mgr.14373) 203 : cephadm 1 Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de, digests ['quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de'] 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[31522]: audit 2024-04-13T07:34:49.694017+0000 mon.a (mon.0) 938 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[31522]: audit 2024-04-13T07:34:49.699342+0000 mon.a (mon.0) 939 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:34:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:50 smithi089 bash[31522]: cephadm 2024-04-13T07:34:49.700140+0000 mgr.y (mgr.14373) 204 : cephadm 1 Upgrade: Need to upgrade myself (mgr.y) 2024-04-13T07:34:52.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:51 smithi203 bash[23480]: cephadm 2024-04-13T07:34:50.115747+0000 mgr.y (mgr.14373) 205 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de on smithi203 2024-04-13T07:34:52.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:51 smithi203 bash[23480]: audit 2024-04-13T07:34:50.784849+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.89:0/1800133346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:52.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:51 smithi089 bash[21353]: cephadm 2024-04-13T07:34:50.115747+0000 mgr.y (mgr.14373) 205 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de on smithi203 2024-04-13T07:34:52.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:51 smithi089 bash[21353]: audit 2024-04-13T07:34:50.784849+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.89:0/1800133346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:52.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:51 smithi089 bash[31522]: cephadm 2024-04-13T07:34:50.115747+0000 mgr.y (mgr.14373) 205 : cephadm 1 Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de on smithi203 2024-04-13T07:34:52.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:51 smithi089 bash[31522]: audit 2024-04-13T07:34:50.784849+0000 mon.c (mon.2) 26 : audit 0 from='client.? 172.21.15.89:0/1800133346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:53.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:52 smithi203 bash[23480]: cluster 2024-04-13T07:34:50.950063+0000 mgr.y (mgr.14373) 206 : cluster 0 pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:53.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:52 smithi089 bash[31522]: cluster 2024-04-13T07:34:50.950063+0000 mgr.y (mgr.14373) 206 : cluster 0 pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:53.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:52 smithi089 bash[21353]: cluster 2024-04-13T07:34:50.950063+0000 mgr.y (mgr.14373) 206 : cluster 0 pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:54.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:53 smithi203 bash[23480]: audit 2024-04-13T07:34:53.217420+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.89:0/3854294661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:54.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:53 smithi089 bash[21353]: audit 2024-04-13T07:34:53.217420+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.89:0/3854294661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:54.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:53 smithi089 bash[31522]: audit 2024-04-13T07:34:53.217420+0000 mon.c (mon.2) 27 : audit 0 from='client.? 172.21.15.89:0/3854294661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:55.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:54 smithi203 bash[23480]: cluster 2024-04-13T07:34:52.950470+0000 mgr.y (mgr.14373) 207 : cluster 0 pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:55.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:54 smithi203 bash[23480]: audit 2024-04-13T07:34:53.936143+0000 mon.a (mon.0) 940 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:55.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:54 smithi089 bash[31522]: cluster 2024-04-13T07:34:52.950470+0000 mgr.y (mgr.14373) 207 : cluster 0 pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:55.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:54 smithi089 bash[31522]: audit 2024-04-13T07:34:53.936143+0000 mon.a (mon.0) 940 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:55.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:54 smithi089 bash[21353]: cluster 2024-04-13T07:34:52.950470+0000 mgr.y (mgr.14373) 207 : cluster 0 pgmap v189: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:55.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:54 smithi089 bash[21353]: audit 2024-04-13T07:34:53.936143+0000 mon.a (mon.0) 940 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:34:56.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:55 smithi203 bash[23480]: audit 2024-04-13T07:34:55.644791+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.89:0/3527983878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:56.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:55 smithi089 bash[21353]: audit 2024-04-13T07:34:55.644791+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.89:0/3527983878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:56.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:55 smithi089 bash[31522]: audit 2024-04-13T07:34:55.644791+0000 mon.c (mon.2) 28 : audit 0 from='client.? 172.21.15.89:0/3527983878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:56.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:34:56 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:34:56] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-13T07:34:57.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:56 smithi203 bash[23480]: cluster 2024-04-13T07:34:54.951474+0000 mgr.y (mgr.14373) 208 : cluster 0 pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:57.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:56 smithi089 bash[21353]: cluster 2024-04-13T07:34:54.951474+0000 mgr.y (mgr.14373) 208 : cluster 0 pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:57.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:56 smithi089 bash[31522]: cluster 2024-04-13T07:34:54.951474+0000 mgr.y (mgr.14373) 208 : cluster 0 pgmap v190: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:59.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:58 smithi203 bash[23480]: cluster 2024-04-13T07:34:56.952112+0000 mgr.y (mgr.14373) 209 : cluster 0 pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:59.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:34:58 smithi203 bash[23480]: audit 2024-04-13T07:34:58.072136+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.89:0/3948267073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:59.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:58 smithi089 bash[31522]: cluster 2024-04-13T07:34:56.952112+0000 mgr.y (mgr.14373) 209 : cluster 0 pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:59.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:34:58 smithi089 bash[31522]: audit 2024-04-13T07:34:58.072136+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.89:0/3948267073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:34:59.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:58 smithi089 bash[21353]: cluster 2024-04-13T07:34:56.952112+0000 mgr.y (mgr.14373) 209 : cluster 0 pgmap v191: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:34:59.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:34:58 smithi089 bash[21353]: audit 2024-04-13T07:34:58.072136+0000 mon.c (mon.2) 29 : audit 0 from='client.? 172.21.15.89:0/3948267073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:00 smithi203 bash[23480]: cluster 2024-04-13T07:34:58.952754+0000 mgr.y (mgr.14373) 210 : cluster 0 pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:00 smithi203 bash[23480]: audit 2024-04-13T07:35:00.510528+0000 mon.a (mon.0) 941 : audit 0 from='client.? 172.21.15.89:0/3068041520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:01.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:00 smithi089 bash[21353]: cluster 2024-04-13T07:34:58.952754+0000 mgr.y (mgr.14373) 210 : cluster 0 pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:01.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:00 smithi089 bash[21353]: audit 2024-04-13T07:35:00.510528+0000 mon.a (mon.0) 941 : audit 0 from='client.? 172.21.15.89:0/3068041520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:01.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:00 smithi089 bash[31522]: cluster 2024-04-13T07:34:58.952754+0000 mgr.y (mgr.14373) 210 : cluster 0 pgmap v192: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:01.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:00 smithi089 bash[31522]: audit 2024-04-13T07:35:00.510528+0000 mon.a (mon.0) 941 : audit 0 from='client.? 172.21.15.89:0/3068041520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:03.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:02 smithi203 bash[23480]: cluster 2024-04-13T07:35:00.953784+0000 mgr.y (mgr.14373) 211 : cluster 0 pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:03.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:02 smithi089 bash[31522]: cluster 2024-04-13T07:35:00.953784+0000 mgr.y (mgr.14373) 211 : cluster 0 pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:03.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:02 smithi089 bash[21353]: cluster 2024-04-13T07:35:00.953784+0000 mgr.y (mgr.14373) 211 : cluster 0 pgmap v193: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:04.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:03 smithi203 bash[23480]: audit 2024-04-13T07:35:02.940415+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.89:0/2999104726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:04.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:03 smithi089 bash[31522]: audit 2024-04-13T07:35:02.940415+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.89:0/2999104726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:04.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:03 smithi089 bash[21353]: audit 2024-04-13T07:35:02.940415+0000 mon.a (mon.0) 942 : audit 0 from='client.? 172.21.15.89:0/2999104726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:05.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:04 smithi203 bash[23480]: cluster 2024-04-13T07:35:02.954370+0000 mgr.y (mgr.14373) 212 : cluster 0 pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:05.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:04 smithi089 bash[31522]: cluster 2024-04-13T07:35:02.954370+0000 mgr.y (mgr.14373) 212 : cluster 0 pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:05.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:04 smithi089 bash[21353]: cluster 2024-04-13T07:35:02.954370+0000 mgr.y (mgr.14373) 212 : cluster 0 pgmap v194: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:06.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:05 smithi203 bash[23480]: audit 2024-04-13T07:35:05.366828+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.89:0/1084022816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:06.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:05 smithi089 bash[21353]: audit 2024-04-13T07:35:05.366828+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.89:0/1084022816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:06.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:05 smithi089 bash[31522]: audit 2024-04-13T07:35:05.366828+0000 mon.c (mon.2) 30 : audit 0 from='client.? 172.21.15.89:0/1084022816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:06.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:06 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:35:06] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-13T07:35:07.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:06 smithi203 bash[23480]: cluster 2024-04-13T07:35:04.955564+0000 mgr.y (mgr.14373) 213 : cluster 0 pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:07.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:06 smithi089 bash[31522]: cluster 2024-04-13T07:35:04.955564+0000 mgr.y (mgr.14373) 213 : cluster 0 pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:07.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:06 smithi089 bash[21353]: cluster 2024-04-13T07:35:04.955564+0000 mgr.y (mgr.14373) 213 : cluster 0 pgmap v195: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:09.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:08 smithi089 bash[31522]: cluster 2024-04-13T07:35:06.956335+0000 mgr.y (mgr.14373) 214 : cluster 0 pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:09.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:08 smithi089 bash[31522]: audit 2024-04-13T07:35:07.801470+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.89:0/2839726637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:09.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:08 smithi089 bash[21353]: cluster 2024-04-13T07:35:06.956335+0000 mgr.y (mgr.14373) 214 : cluster 0 pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:09.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:08 smithi089 bash[21353]: audit 2024-04-13T07:35:07.801470+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.89:0/2839726637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:09.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:08 smithi203 bash[23480]: cluster 2024-04-13T07:35:06.956335+0000 mgr.y (mgr.14373) 214 : cluster 0 pgmap v196: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:09.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:08 smithi203 bash[23480]: audit 2024-04-13T07:35:07.801470+0000 mon.c (mon.2) 31 : audit 0 from='client.? 172.21.15.89:0/2839726637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:10.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:09 smithi089 bash[21353]: audit 2024-04-13T07:35:08.936563+0000 mon.a (mon.0) 943 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:10.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:09 smithi089 bash[31522]: audit 2024-04-13T07:35:08.936563+0000 mon.a (mon.0) 943 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:10.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:09 smithi203 bash[23480]: audit 2024-04-13T07:35:08.936563+0000 mon.a (mon.0) 943 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:11.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:10 smithi089 bash[21353]: cluster 2024-04-13T07:35:08.956972+0000 mgr.y (mgr.14373) 215 : cluster 0 pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:11.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:10 smithi089 bash[21353]: audit 2024-04-13T07:35:10.236458+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.89:0/4115508262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:11.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:10 smithi089 bash[31522]: cluster 2024-04-13T07:35:08.956972+0000 mgr.y (mgr.14373) 215 : cluster 0 pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:11.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:10 smithi089 bash[31522]: audit 2024-04-13T07:35:10.236458+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.89:0/4115508262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:11.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:10 smithi203 bash[23480]: cluster 2024-04-13T07:35:08.956972+0000 mgr.y (mgr.14373) 215 : cluster 0 pgmap v197: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:11.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:10 smithi203 bash[23480]: audit 2024-04-13T07:35:10.236458+0000 mon.c (mon.2) 32 : audit 0 from='client.? 172.21.15.89:0/4115508262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:13.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:12 smithi089 bash[31522]: cluster 2024-04-13T07:35:10.958088+0000 mgr.y (mgr.14373) 216 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:13.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:12 smithi089 bash[31522]: audit 2024-04-13T07:35:12.666529+0000 mon.a (mon.0) 944 : audit 0 from='client.? 172.21.15.89:0/2979328172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:13.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:12 smithi089 bash[21353]: cluster 2024-04-13T07:35:10.958088+0000 mgr.y (mgr.14373) 216 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:13.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:12 smithi089 bash[21353]: audit 2024-04-13T07:35:12.666529+0000 mon.a (mon.0) 944 : audit 0 from='client.? 172.21.15.89:0/2979328172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:13.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:12 smithi203 bash[23480]: cluster 2024-04-13T07:35:10.958088+0000 mgr.y (mgr.14373) 216 : cluster 0 pgmap v198: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:13.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:12 smithi203 bash[23480]: audit 2024-04-13T07:35:12.666529+0000 mon.a (mon.0) 944 : audit 0 from='client.? 172.21.15.89:0/2979328172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:15.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:14 smithi089 bash[31522]: cluster 2024-04-13T07:35:12.958697+0000 mgr.y (mgr.14373) 217 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:15.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:14 smithi089 bash[21353]: cluster 2024-04-13T07:35:12.958697+0000 mgr.y (mgr.14373) 217 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:15.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:14 smithi203 bash[23480]: cluster 2024-04-13T07:35:12.958697+0000 mgr.y (mgr.14373) 217 : cluster 0 pgmap v199: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:16.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:15 smithi089 bash[21353]: audit 2024-04-13T07:35:15.105908+0000 mon.a (mon.0) 945 : audit 0 from='client.? 172.21.15.89:0/4211212213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:16.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:15 smithi089 bash[31522]: audit 2024-04-13T07:35:15.105908+0000 mon.a (mon.0) 945 : audit 0 from='client.? 172.21.15.89:0/4211212213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:16.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:15 smithi203 bash[23480]: audit 2024-04-13T07:35:15.105908+0000 mon.a (mon.0) 945 : audit 0 from='client.? 172.21.15.89:0/4211212213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:16.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:16 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:35:16] "GET /metrics HTTP/1.1" 200 33827 "" "Prometheus/2.43.0" 2024-04-13T07:35:17.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:16 smithi089 bash[21353]: cluster 2024-04-13T07:35:14.959733+0000 mgr.y (mgr.14373) 218 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:17.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:16 smithi089 bash[31522]: cluster 2024-04-13T07:35:14.959733+0000 mgr.y (mgr.14373) 218 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:17.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:16 smithi203 bash[23480]: cluster 2024-04-13T07:35:14.959733+0000 mgr.y (mgr.14373) 218 : cluster 0 pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:18.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:17 smithi089 bash[31522]: audit 2024-04-13T07:35:17.539176+0000 mon.a (mon.0) 946 : audit 0 from='client.? 172.21.15.89:0/2030242728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:18.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:17 smithi089 bash[21353]: audit 2024-04-13T07:35:17.539176+0000 mon.a (mon.0) 946 : audit 0 from='client.? 172.21.15.89:0/2030242728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:18.261 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:17 smithi203 bash[23480]: audit 2024-04-13T07:35:17.539176+0000 mon.a (mon.0) 946 : audit 0 from='client.? 172.21.15.89:0/2030242728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:19.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:18 smithi203 bash[23480]: cluster 2024-04-13T07:35:16.960282+0000 mgr.y (mgr.14373) 219 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:19.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:18 smithi089 bash[31522]: cluster 2024-04-13T07:35:16.960282+0000 mgr.y (mgr.14373) 219 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:19.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:18 smithi089 bash[21353]: cluster 2024-04-13T07:35:16.960282+0000 mgr.y (mgr.14373) 219 : cluster 0 pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:21.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:20 smithi203 bash[23480]: cluster 2024-04-13T07:35:18.960835+0000 mgr.y (mgr.14373) 220 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:21.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:20 smithi203 bash[23480]: audit 2024-04-13T07:35:19.981387+0000 mon.a (mon.0) 947 : audit 0 from='client.? 172.21.15.89:0/1486261542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:21.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:20 smithi089 bash[21353]: cluster 2024-04-13T07:35:18.960835+0000 mgr.y (mgr.14373) 220 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:21.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:20 smithi089 bash[21353]: audit 2024-04-13T07:35:19.981387+0000 mon.a (mon.0) 947 : audit 0 from='client.? 172.21.15.89:0/1486261542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:21.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:20 smithi089 bash[31522]: cluster 2024-04-13T07:35:18.960835+0000 mgr.y (mgr.14373) 220 : cluster 0 pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:21.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:20 smithi089 bash[31522]: audit 2024-04-13T07:35:19.981387+0000 mon.a (mon.0) 947 : audit 0 from='client.? 172.21.15.89:0/1486261542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:23.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:22 smithi203 bash[23480]: cluster 2024-04-13T07:35:20.961922+0000 mgr.y (mgr.14373) 221 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:23.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:22 smithi203 bash[23480]: audit 2024-04-13T07:35:22.414644+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.89:0/2180673787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:23.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:22 smithi089 bash[21353]: cluster 2024-04-13T07:35:20.961922+0000 mgr.y (mgr.14373) 221 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:23.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:22 smithi089 bash[21353]: audit 2024-04-13T07:35:22.414644+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.89:0/2180673787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:23.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:22 smithi089 bash[31522]: cluster 2024-04-13T07:35:20.961922+0000 mgr.y (mgr.14373) 221 : cluster 0 pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:23.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:22 smithi089 bash[31522]: audit 2024-04-13T07:35:22.414644+0000 mon.c (mon.2) 33 : audit 0 from='client.? 172.21.15.89:0/2180673787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:25.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:24 smithi203 bash[23480]: cluster 2024-04-13T07:35:22.962538+0000 mgr.y (mgr.14373) 222 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:25.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:24 smithi203 bash[23480]: audit 2024-04-13T07:35:23.936889+0000 mon.a (mon.0) 948 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:25.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:24 smithi089 bash[21353]: cluster 2024-04-13T07:35:22.962538+0000 mgr.y (mgr.14373) 222 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:25.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:24 smithi089 bash[21353]: audit 2024-04-13T07:35:23.936889+0000 mon.a (mon.0) 948 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:25.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:24 smithi089 bash[31522]: cluster 2024-04-13T07:35:22.962538+0000 mgr.y (mgr.14373) 222 : cluster 0 pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:25.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:24 smithi089 bash[31522]: audit 2024-04-13T07:35:23.936889+0000 mon.a (mon.0) 948 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:26.168 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:25 smithi089 bash[31522]: audit 2024-04-13T07:35:24.857070+0000 mon.a (mon.0) 949 : audit 0 from='client.? 172.21.15.89:0/2179740797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:26.168 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:25 smithi089 bash[21353]: audit 2024-04-13T07:35:24.857070+0000 mon.a (mon.0) 949 : audit 0 from='client.? 172.21.15.89:0/2179740797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:26.261 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:25 smithi203 bash[23480]: audit 2024-04-13T07:35:24.857070+0000 mon.a (mon.0) 949 : audit 0 from='client.? 172.21.15.89:0/2179740797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:26.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:26 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:35:26] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-13T07:35:27.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:26 smithi203 bash[23480]: cluster 2024-04-13T07:35:24.963606+0000 mgr.y (mgr.14373) 223 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:27.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:26 smithi089 bash[31522]: cluster 2024-04-13T07:35:24.963606+0000 mgr.y (mgr.14373) 223 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:27.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:26 smithi089 bash[21353]: cluster 2024-04-13T07:35:24.963606+0000 mgr.y (mgr.14373) 223 : cluster 0 pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:28.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:27 smithi203 bash[23480]: audit 2024-04-13T07:35:27.281428+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.89:0/3252128763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:28.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:27 smithi089 bash[31522]: audit 2024-04-13T07:35:27.281428+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.89:0/3252128763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:28.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:27 smithi089 bash[21353]: audit 2024-04-13T07:35:27.281428+0000 mon.c (mon.2) 34 : audit 0 from='client.? 172.21.15.89:0/3252128763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:29.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:28 smithi203 bash[23480]: cluster 2024-04-13T07:35:26.964201+0000 mgr.y (mgr.14373) 224 : cluster 0 pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:29.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:28 smithi089 bash[31522]: cluster 2024-04-13T07:35:26.964201+0000 mgr.y (mgr.14373) 224 : cluster 0 pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:29.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:28 smithi089 bash[21353]: cluster 2024-04-13T07:35:26.964201+0000 mgr.y (mgr.14373) 224 : cluster 0 pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:30.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:29 smithi203 bash[23480]: audit 2024-04-13T07:35:29.727725+0000 mon.a (mon.0) 950 : audit 0 from='client.? 172.21.15.89:0/414849935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:30.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:29 smithi089 bash[21353]: audit 2024-04-13T07:35:29.727725+0000 mon.a (mon.0) 950 : audit 0 from='client.? 172.21.15.89:0/414849935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:30.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:29 smithi089 bash[31522]: audit 2024-04-13T07:35:29.727725+0000 mon.a (mon.0) 950 : audit 0 from='client.? 172.21.15.89:0/414849935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:31.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:30 smithi203 bash[23480]: cluster 2024-04-13T07:35:28.964818+0000 mgr.y (mgr.14373) 225 : cluster 0 pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:31.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:30 smithi089 bash[31522]: cluster 2024-04-13T07:35:28.964818+0000 mgr.y (mgr.14373) 225 : cluster 0 pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:31.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:30 smithi089 bash[21353]: cluster 2024-04-13T07:35:28.964818+0000 mgr.y (mgr.14373) 225 : cluster 0 pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:32.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:31 smithi089 bash[21353]: cluster 2024-04-13T07:35:30.965940+0000 mgr.y (mgr.14373) 226 : cluster 0 pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:32.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:31 smithi089 bash[31522]: cluster 2024-04-13T07:35:30.965940+0000 mgr.y (mgr.14373) 226 : cluster 0 pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:32.386 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:31 smithi203 bash[23480]: cluster 2024-04-13T07:35:30.965940+0000 mgr.y (mgr.14373) 226 : cluster 0 pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:33.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:32 smithi203 bash[23480]: audit 2024-04-13T07:35:32.173511+0000 mon.a (mon.0) 951 : audit 0 from='client.? 172.21.15.89:0/1854961395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:33.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:32 smithi089 bash[21353]: audit 2024-04-13T07:35:32.173511+0000 mon.a (mon.0) 951 : audit 0 from='client.? 172.21.15.89:0/1854961395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:33.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:32 smithi089 bash[31522]: audit 2024-04-13T07:35:32.173511+0000 mon.a (mon.0) 951 : audit 0 from='client.? 172.21.15.89:0/1854961395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:34.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:33 smithi203 bash[23480]: cluster 2024-04-13T07:35:32.966557+0000 mgr.y (mgr.14373) 227 : cluster 0 pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:34.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:33 smithi089 bash[21353]: cluster 2024-04-13T07:35:32.966557+0000 mgr.y (mgr.14373) 227 : cluster 0 pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:34.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:33 smithi089 bash[31522]: cluster 2024-04-13T07:35:32.966557+0000 mgr.y (mgr.14373) 227 : cluster 0 pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:35.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:34 smithi203 bash[23480]: audit 2024-04-13T07:35:34.599305+0000 mon.a (mon.0) 952 : audit 0 from='client.? 172.21.15.89:0/1968424905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:35.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:34 smithi089 bash[21353]: audit 2024-04-13T07:35:34.599305+0000 mon.a (mon.0) 952 : audit 0 from='client.? 172.21.15.89:0/1968424905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:35.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:34 smithi089 bash[31522]: audit 2024-04-13T07:35:34.599305+0000 mon.a (mon.0) 952 : audit 0 from='client.? 172.21.15.89:0/1968424905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:36.168 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:35 smithi089 bash[31522]: cluster 2024-04-13T07:35:34.967623+0000 mgr.y (mgr.14373) 228 : cluster 0 pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:36.168 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:35 smithi089 bash[21353]: cluster 2024-04-13T07:35:34.967623+0000 mgr.y (mgr.14373) 228 : cluster 0 pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:36.261 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:35 smithi203 bash[23480]: cluster 2024-04-13T07:35:34.967623+0000 mgr.y (mgr.14373) 228 : cluster 0 pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:36.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:36 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:35:36] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-13T07:35:38.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:38 smithi089 bash[31522]: cluster 2024-04-13T07:35:36.968163+0000 mgr.y (mgr.14373) 229 : cluster 0 pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:38.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:38 smithi089 bash[31522]: audit 2024-04-13T07:35:37.044545+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.89:0/1141552612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:38.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:38 smithi089 bash[21353]: cluster 2024-04-13T07:35:36.968163+0000 mgr.y (mgr.14373) 229 : cluster 0 pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:38.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:38 smithi089 bash[21353]: audit 2024-04-13T07:35:37.044545+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.89:0/1141552612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:38.347 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:38 smithi203 bash[23480]: cluster 2024-04-13T07:35:36.968163+0000 mgr.y (mgr.14373) 229 : cluster 0 pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:38.347 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:38 smithi203 bash[23480]: audit 2024-04-13T07:35:37.044545+0000 mon.c (mon.2) 35 : audit 0 from='client.? 172.21.15.89:0/1141552612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:39.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[21353]: cephadm 2024-04-13T07:35:38.038746+0000 mgr.y (mgr.14373) 230 : cephadm 1 Upgrade: Updating mgr.x 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[21353]: audit 2024-04-13T07:35:38.047664+0000 mon.a (mon.0) 953 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[21353]: audit 2024-04-13T07:35:38.049739+0000 mon.a (mon.0) 954 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[21353]: audit 2024-04-13T07:35:38.051510+0000 mon.a (mon.0) 955 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[21353]: audit 2024-04-13T07:35:38.053385+0000 mon.a (mon.0) 956 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[21353]: cephadm 2024-04-13T07:35:38.055352+0000 mgr.y (mgr.14373) 231 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[21353]: audit 2024-04-13T07:35:38.937232+0000 mon.a (mon.0) 957 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[31522]: cephadm 2024-04-13T07:35:38.038746+0000 mgr.y (mgr.14373) 230 : cephadm 1 Upgrade: Updating mgr.x 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[31522]: audit 2024-04-13T07:35:38.047664+0000 mon.a (mon.0) 953 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[31522]: audit 2024-04-13T07:35:38.049739+0000 mon.a (mon.0) 954 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[31522]: audit 2024-04-13T07:35:38.051510+0000 mon.a (mon.0) 955 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:35:39.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[31522]: audit 2024-04-13T07:35:38.053385+0000 mon.a (mon.0) 956 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:35:39.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[31522]: cephadm 2024-04-13T07:35:38.055352+0000 mgr.y (mgr.14373) 231 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:35:39.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:39 smithi089 bash[31522]: audit 2024-04-13T07:35:38.937232+0000 mon.a (mon.0) 957 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:39.318 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:39 smithi203 bash[23480]: cephadm 2024-04-13T07:35:38.038746+0000 mgr.y (mgr.14373) 230 : cephadm 1 Upgrade: Updating mgr.x 2024-04-13T07:35:39.318 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:39 smithi203 bash[23480]: audit 2024-04-13T07:35:38.047664+0000 mon.a (mon.0) 953 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:39.318 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:39 smithi203 bash[23480]: audit 2024-04-13T07:35:38.049739+0000 mon.a (mon.0) 954 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:35:39.318 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:39 smithi203 bash[23480]: audit 2024-04-13T07:35:38.051510+0000 mon.a (mon.0) 955 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:35:39.318 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:39 smithi203 bash[23480]: audit 2024-04-13T07:35:38.053385+0000 mon.a (mon.0) 956 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:35:39.318 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:39 smithi203 bash[23480]: cephadm 2024-04-13T07:35:38.055352+0000 mgr.y (mgr.14373) 231 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:35:39.318 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:39 smithi203 bash[23480]: audit 2024-04-13T07:35:38.937232+0000 mon.a (mon.0) 957 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:40.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:40 smithi089 bash[21353]: cluster 2024-04-13T07:35:38.974003+0000 mgr.y (mgr.14373) 232 : cluster 0 pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:40.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:40 smithi089 bash[21353]: audit 2024-04-13T07:35:39.483033+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.89:0/732176862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:40.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:40 smithi089 bash[31522]: cluster 2024-04-13T07:35:38.974003+0000 mgr.y (mgr.14373) 232 : cluster 0 pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:40.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:40 smithi089 bash[31522]: audit 2024-04-13T07:35:39.483033+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.89:0/732176862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:40.403 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:40 smithi203 bash[23480]: cluster 2024-04-13T07:35:38.974003+0000 mgr.y (mgr.14373) 232 : cluster 0 pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:40.403 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:40 smithi203 bash[23480]: audit 2024-04-13T07:35:39.483033+0000 mon.c (mon.2) 36 : audit 0 from='client.? 172.21.15.89:0/732176862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:40.673 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:40.673 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:40.673 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:40.674 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:40.674 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:40.675 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:40.675 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: Stopping Ceph mgr.x for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:35:40.675 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:40.675 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:40.675 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:35:40 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:41.665 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:41 smithi203 bash[52163]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mgr-x 2024-04-13T07:35:41.665 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-13T07:35:41.665 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:41 smithi203 bash[52227]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mgr-x 2024-04-13T07:35:41.665 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-13T07:35:41.665 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: Stopped Ceph mgr.x for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:35:42.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:42.012 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:42.012 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:42.012 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:42.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:42.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: Started Ceph mgr.x for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:35:42.013 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:42.013 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:42.014 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:42.014 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:35:41 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:24: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:35:43.141 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:42 smithi203 bash[23480]: cluster 2024-04-13T07:35:40.975130+0000 mgr.y (mgr.14373) 233 : cluster 0 pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:43.141 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:42 smithi203 bash[23480]: audit 2024-04-13T07:35:41.824272+0000 mon.a (mon.0) 958 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:43.142 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:42 smithi203 bash[23480]: audit 2024-04-13T07:35:41.835359+0000 mon.a (mon.0) 959 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:43.142 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:42 smithi203 bash[23480]: audit 2024-04-13T07:35:41.886563+0000 mon.a (mon.0) 960 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:35:43.142 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:42 smithi203 bash[23480]: audit 2024-04-13T07:35:41.927334+0000 mon.a (mon.0) 961 : audit 0 from='client.? 172.21.15.89:0/1502579930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:43.142 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:43 smithi203 bash[52286]: debug 2024-04-13T07:35:43.000+0000 7f8233505700 1 -- 172.21.15.203:0/838743329 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d139dc05a0 con 0x55d13ab40800 2024-04-13T07:35:43.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[21353]: cluster 2024-04-13T07:35:40.975130+0000 mgr.y (mgr.14373) 233 : cluster 0 pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:43.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[21353]: audit 2024-04-13T07:35:41.824272+0000 mon.a (mon.0) 958 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:43.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[21353]: audit 2024-04-13T07:35:41.835359+0000 mon.a (mon.0) 959 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:43.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[21353]: audit 2024-04-13T07:35:41.886563+0000 mon.a (mon.0) 960 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:35:43.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[21353]: audit 2024-04-13T07:35:41.927334+0000 mon.a (mon.0) 961 : audit 0 from='client.? 172.21.15.89:0/1502579930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[31522]: cluster 2024-04-13T07:35:40.975130+0000 mgr.y (mgr.14373) 233 : cluster 0 pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[31522]: audit 2024-04-13T07:35:41.824272+0000 mon.a (mon.0) 958 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[31522]: audit 2024-04-13T07:35:41.835359+0000 mon.a (mon.0) 959 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[31522]: audit 2024-04-13T07:35:41.886563+0000 mon.a (mon.0) 960 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:35:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:42 smithi089 bash[31522]: audit 2024-04-13T07:35:41.927334+0000 mon.a (mon.0) 961 : audit 0 from='client.? 172.21.15.89:0/1502579930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:43.511 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:43 smithi203 bash[52286]: debug 2024-04-13T07:35:43.148+0000 7f823d715200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:35:43.798 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:35:43 smithi089 bash[58201]: ts=2024-04-13T07:35:43.514Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:35:45.011 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:44 smithi203 bash[52286]: debug 2024-04-13T07:35:44.719+0000 7f823d715200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:35:45.011 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:44 smithi203 bash[52286]: debug 2024-04-13T07:35:44.819+0000 7f823d715200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:35:45.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:44 smithi203 bash[23480]: cluster 2024-04-13T07:35:42.975690+0000 mgr.y (mgr.14373) 234 : cluster 0 pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:45.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:44 smithi203 bash[23480]: audit 2024-04-13T07:35:44.364181+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.89:0/2817531890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:45.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:44 smithi089 bash[21353]: cluster 2024-04-13T07:35:42.975690+0000 mgr.y (mgr.14373) 234 : cluster 0 pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:45.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:44 smithi089 bash[21353]: audit 2024-04-13T07:35:44.364181+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.89:0/2817531890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:45.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:44 smithi089 bash[31522]: cluster 2024-04-13T07:35:42.975690+0000 mgr.y (mgr.14373) 234 : cluster 0 pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:45.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:44 smithi089 bash[31522]: audit 2024-04-13T07:35:44.364181+0000 mon.c (mon.2) 37 : audit 0 from='client.? 172.21.15.89:0/2817531890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:45.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:45 smithi203 bash[52286]: debug 2024-04-13T07:35:45.067+0000 7f823d715200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:35:46.261 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:45 smithi203 bash[52286]: debug 2024-04-13T07:35:45.871+0000 7f823d715200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:35:46.261 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:45 smithi203 bash[52286]: debug 2024-04-13T07:35:45.979+0000 7f823d715200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:35:46.261 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:46 smithi203 bash[52286]: debug 2024-04-13T07:35:46.083+0000 7f823d715200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:35:46.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:46 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:35:46] "GET /metrics HTTP/1.1" 200 33827 "" "Prometheus/2.43.0" 2024-04-13T07:35:46.761 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:46 smithi203 bash[52286]: debug 2024-04-13T07:35:46.443+0000 7f823d715200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:35:46.761 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:46 smithi203 bash[52286]: debug 2024-04-13T07:35:46.555+0000 7f823d715200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:35:46.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:46 smithi203 bash[52286]: debug 2024-04-13T07:35:46.659+0000 7f823d715200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:35:47.261 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:47 smithi203 bash[52286]: debug 2024-04-13T07:35:47.047+0000 7f823d715200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:35:47.261 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:46 smithi203 bash[23480]: cluster 2024-04-13T07:35:44.976834+0000 mgr.y (mgr.14373) 235 : cluster 0 pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:47.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:46 smithi203 bash[23480]: audit 2024-04-13T07:35:46.801122+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.89:0/109007880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:47.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:46 smithi089 bash[31522]: cluster 2024-04-13T07:35:44.976834+0000 mgr.y (mgr.14373) 235 : cluster 0 pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:47.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:46 smithi089 bash[31522]: audit 2024-04-13T07:35:46.801122+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.89:0/109007880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:47.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:46 smithi089 bash[21353]: cluster 2024-04-13T07:35:44.976834+0000 mgr.y (mgr.14373) 235 : cluster 0 pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:47.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:46 smithi089 bash[21353]: audit 2024-04-13T07:35:46.801122+0000 mon.c (mon.2) 38 : audit 0 from='client.? 172.21.15.89:0/109007880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:47.610 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:47 smithi203 bash[52286]: debug 2024-04-13T07:35:47.311+0000 7f823d715200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:35:48.011 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:47 smithi203 bash[52286]: debug 2024-04-13T07:35:47.615+0000 7f823d715200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:35:48.011 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:47 smithi203 bash[52286]: debug 2024-04-13T07:35:47.759+0000 7f823d715200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:35:48.319 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[52286]: debug 2024-04-13T07:35:48.047+0000 7f823d715200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:35:48.717 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[52286]: debug 2024-04-13T07:35:48.475+0000 7f823d715200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:35:48.718 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[23480]: cluster 2024-04-13T07:35:46.977654+0000 mgr.y (mgr.14373) 236 : cluster 0 pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:48.718 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[23480]: audit 2024-04-13T07:35:47.324759+0000 mon.a (mon.0) 962 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.718 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[23480]: audit 2024-04-13T07:35:47.331579+0000 mon.a (mon.0) 963 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.718 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[23480]: audit 2024-04-13T07:35:47.781061+0000 mon.a (mon.0) 964 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.718 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[23480]: audit 2024-04-13T07:35:47.792862+0000 mon.a (mon.0) 965 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[21353]: cluster 2024-04-13T07:35:46.977654+0000 mgr.y (mgr.14373) 236 : cluster 0 pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[21353]: audit 2024-04-13T07:35:47.324759+0000 mon.a (mon.0) 962 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[21353]: audit 2024-04-13T07:35:47.331579+0000 mon.a (mon.0) 963 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[21353]: audit 2024-04-13T07:35:47.781061+0000 mon.a (mon.0) 964 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[21353]: audit 2024-04-13T07:35:47.792862+0000 mon.a (mon.0) 965 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[31522]: cluster 2024-04-13T07:35:46.977654+0000 mgr.y (mgr.14373) 236 : cluster 0 pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[31522]: audit 2024-04-13T07:35:47.324759+0000 mon.a (mon.0) 962 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[31522]: audit 2024-04-13T07:35:47.331579+0000 mon.a (mon.0) 963 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[31522]: audit 2024-04-13T07:35:47.781061+0000 mon.a (mon.0) 964 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:48 smithi089 bash[31522]: audit 2024-04-13T07:35:47.792862+0000 mon.a (mon.0) 965 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:49.011 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[52286]: debug 2024-04-13T07:35:48.723+0000 7f823d715200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:35:49.011 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:48 smithi203 bash[52286]: debug 2024-04-13T07:35:48.847+0000 7f823d715200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:35:49.511 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:49 smithi203 bash[52286]: debug 2024-04-13T07:35:49.115+0000 7f823d715200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:35:50.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:49 smithi203 bash[23480]: audit 2024-04-13T07:35:48.644489+0000 mon.a (mon.0) 966 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:50.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:49 smithi203 bash[23480]: audit 2024-04-13T07:35:48.656477+0000 mon.a (mon.0) 967 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:50.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:49 smithi203 bash[23480]: audit 2024-04-13T07:35:49.239376+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.89:0/2291262224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:50.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:49 smithi089 bash[31522]: audit 2024-04-13T07:35:48.644489+0000 mon.a (mon.0) 966 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:50.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:49 smithi089 bash[31522]: audit 2024-04-13T07:35:48.656477+0000 mon.a (mon.0) 967 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:50.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:49 smithi089 bash[31522]: audit 2024-04-13T07:35:49.239376+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.89:0/2291262224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:50.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:49 smithi089 bash[21353]: audit 2024-04-13T07:35:48.644489+0000 mon.a (mon.0) 966 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:50.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:49 smithi089 bash[21353]: audit 2024-04-13T07:35:48.656477+0000 mon.a (mon.0) 967 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:50.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:49 smithi089 bash[21353]: audit 2024-04-13T07:35:49.239376+0000 mon.c (mon.2) 39 : audit 0 from='client.? 172.21.15.89:0/2291262224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:50.511 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:50 smithi203 bash[52286]: debug 2024-04-13T07:35:50.151+0000 7f823d715200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:35:50.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:50 smithi203 bash[52286]: debug 2024-04-13T07:35:50.271+0000 7f823d715200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:35:51.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:50 smithi203 bash[23480]: cluster 2024-04-13T07:35:48.978025+0000 mgr.y (mgr.14373) 237 : cluster 0 pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:51.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:50 smithi203 bash[52286]: debug 2024-04-13T07:35:50.567+0000 7f823d715200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:35:51.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:50 smithi203 bash[52286]: debug 2024-04-13T07:35:50.683+0000 7f823d715200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:35:51.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:50 smithi089 bash[21353]: cluster 2024-04-13T07:35:48.978025+0000 mgr.y (mgr.14373) 237 : cluster 0 pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:51.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:50 smithi089 bash[31522]: cluster 2024-04-13T07:35:48.978025+0000 mgr.y (mgr.14373) 237 : cluster 0 pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:51.388 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[52286]: debug 2024-04-13T07:35:51.139+0000 7f823d715200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:35:51.388 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[52286]: debug 2024-04-13T07:35:51.259+0000 7f823d715200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:35:51.388 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[52286]: [13/Apr/2024:07:35:51] ENGINE Bus STARTING 2024-04-13T07:35:51.388 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[52286]: CherryPy Checker: 2024-04-13T07:35:51.388 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[52286]: The Application mounted at '' has an empty config. 2024-04-13T07:35:51.388 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[52286]: [13/Apr/2024:07:35:51] ENGINE Serving on http://:::9283 2024-04-13T07:35:52.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[52286]: [13/Apr/2024:07:35:51] ENGINE Bus STARTED 2024-04-13T07:35:52.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[23480]: cluster 2024-04-13T07:35:51.282754+0000 mon.a (mon.0) 968 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:35:52.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[23480]: cluster 2024-04-13T07:35:51.282941+0000 mon.a (mon.0) 969 : cluster 0 Standby manager daemon x started 2024-04-13T07:35:52.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[23480]: audit 2024-04-13T07:35:51.290585+0000 mon.b (mon.1) 30 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:35:52.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[23480]: audit 2024-04-13T07:35:51.291576+0000 mon.b (mon.1) 31 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:35:52.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[23480]: audit 2024-04-13T07:35:51.293450+0000 mon.b (mon.1) 32 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:35:52.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:51 smithi203 bash[23480]: audit 2024-04-13T07:35:51.294288+0000 mon.b (mon.1) 33 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:35:52.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[21353]: cluster 2024-04-13T07:35:51.282754+0000 mon.a (mon.0) 968 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:35:52.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[21353]: cluster 2024-04-13T07:35:51.282941+0000 mon.a (mon.0) 969 : cluster 0 Standby manager daemon x started 2024-04-13T07:35:52.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[21353]: audit 2024-04-13T07:35:51.290585+0000 mon.b (mon.1) 30 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[21353]: audit 2024-04-13T07:35:51.291576+0000 mon.b (mon.1) 31 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[21353]: audit 2024-04-13T07:35:51.293450+0000 mon.b (mon.1) 32 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[21353]: audit 2024-04-13T07:35:51.294288+0000 mon.b (mon.1) 33 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[31522]: cluster 2024-04-13T07:35:51.282754+0000 mon.a (mon.0) 968 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[31522]: cluster 2024-04-13T07:35:51.282941+0000 mon.a (mon.0) 969 : cluster 0 Standby manager daemon x started 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[31522]: audit 2024-04-13T07:35:51.290585+0000 mon.b (mon.1) 30 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[31522]: audit 2024-04-13T07:35:51.291576+0000 mon.b (mon.1) 31 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[31522]: audit 2024-04-13T07:35:51.293450+0000 mon.b (mon.1) 32 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:35:52.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:51 smithi089 bash[31522]: audit 2024-04-13T07:35:51.294288+0000 mon.b (mon.1) 33 : audit 0 from='mgr.? 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:35:52.548 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:35:52 smithi089 bash[58201]: ts=2024-04-13T07:35:52.083Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=7 2024-04-13T07:35:53.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:52 smithi203 bash[23480]: cluster 2024-04-13T07:35:50.979218+0000 mgr.y (mgr.14373) 238 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:53.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:52 smithi203 bash[23480]: cluster 2024-04-13T07:35:51.675107+0000 mon.a (mon.0) 970 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-04-13T07:35:53.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:52 smithi203 bash[23480]: audit 2024-04-13T07:35:51.675275+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.89:0/1022526877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:53.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:52 smithi089 bash[21353]: cluster 2024-04-13T07:35:50.979218+0000 mgr.y (mgr.14373) 238 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:53.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:52 smithi089 bash[21353]: cluster 2024-04-13T07:35:51.675107+0000 mon.a (mon.0) 970 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-04-13T07:35:53.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:52 smithi089 bash[21353]: audit 2024-04-13T07:35:51.675275+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.89:0/1022526877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:53.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:52 smithi089 bash[31522]: cluster 2024-04-13T07:35:50.979218+0000 mgr.y (mgr.14373) 238 : cluster 0 pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:53.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:52 smithi089 bash[31522]: cluster 2024-04-13T07:35:51.675107+0000 mon.a (mon.0) 970 : cluster 0 mgrmap e21: y(active, since 6m), standbys: x 2024-04-13T07:35:53.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:52 smithi089 bash[31522]: audit 2024-04-13T07:35:51.675275+0000 mon.a (mon.0) 971 : audit 0 from='client.? 172.21.15.89:0/1022526877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:54.985 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:54 smithi203 bash[23480]: cluster 2024-04-13T07:35:52.979878+0000 mgr.y (mgr.14373) 239 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:54.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:54 smithi203 bash[23480]: audit 2024-04-13T07:35:53.937622+0000 mon.a (mon.0) 972 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:54.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:54 smithi203 bash[23480]: audit 2024-04-13T07:35:54.114613+0000 mon.a (mon.0) 973 : audit 0 from='client.? 172.21.15.89:0/2354190070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:55.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:54 smithi089 bash[21353]: cluster 2024-04-13T07:35:52.979878+0000 mgr.y (mgr.14373) 239 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:55.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:54 smithi089 bash[21353]: audit 2024-04-13T07:35:53.937622+0000 mon.a (mon.0) 972 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:55.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:54 smithi089 bash[21353]: audit 2024-04-13T07:35:54.114613+0000 mon.a (mon.0) 973 : audit 0 from='client.? 172.21.15.89:0/2354190070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:55.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:54 smithi089 bash[31522]: cluster 2024-04-13T07:35:52.979878+0000 mgr.y (mgr.14373) 239 : cluster 0 pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:55.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:54 smithi089 bash[31522]: audit 2024-04-13T07:35:53.937622+0000 mon.a (mon.0) 972 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:35:55.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:54 smithi089 bash[31522]: audit 2024-04-13T07:35:54.114613+0000 mon.a (mon.0) 973 : audit 0 from='client.? 172.21.15.89:0/2354190070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:56.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:56 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:35:56] "GET /metrics HTTP/1.1" 200 33824 "" "Prometheus/2.43.0" 2024-04-13T07:35:57.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:56 smithi203 bash[23480]: cluster 2024-04-13T07:35:54.980933+0000 mgr.y (mgr.14373) 240 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:57.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:56 smithi203 bash[23480]: audit 2024-04-13T07:35:56.553936+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.89:0/3955475172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:57.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:56 smithi089 bash[31522]: cluster 2024-04-13T07:35:54.980933+0000 mgr.y (mgr.14373) 240 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:57.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:56 smithi089 bash[31522]: audit 2024-04-13T07:35:56.553936+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.89:0/3955475172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:56 smithi089 bash[21353]: cluster 2024-04-13T07:35:54.980933+0000 mgr.y (mgr.14373) 240 : cluster 0 pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:56 smithi089 bash[21353]: audit 2024-04-13T07:35:56.553936+0000 mon.c (mon.2) 40 : audit 0 from='client.? 172.21.15.89:0/3955475172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:35:58.832 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21606]: debug 2024-04-13T07:35:58.583+0000 7f0119c00700 -1 mgr handle_mgr_map I was active but no longer am 2024-04-13T07:35:58.832 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21606]: ignoring --setuser ceph since I am not root 2024-04-13T07:35:58.832 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21606]: ignoring --setgroup ceph since I am not root 2024-04-13T07:35:58.833 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21606]: debug 2024-04-13T07:35:58.807+0000 7fe0f406a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:35:58.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: cluster 2024-04-13T07:35:56.981530+0000 mgr.y (mgr.14373) 241 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:58.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.458005+0000 mon.a (mon.0) 974 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:58.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.471583+0000 mon.a (mon.0) 975 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:58.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.474125+0000 mon.a (mon.0) 976 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:35:58.833 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.475968+0000 mon.a (mon.0) 977 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.486262+0000 mon.a (mon.0) 978 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.574122+0000 mon.a (mon.0) 979 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.581672+0000 mon.a (mon.0) 980 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: cluster 2024-04-13T07:35:58.582235+0000 mon.a (mon.0) 981 : cluster 1 Activating manager daemon x 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: cluster 2024-04-13T07:35:58.591295+0000 mon.a (mon.0) 982 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.591869+0000 mon.a (mon.0) 983 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: cluster 2024-04-13T07:35:58.591957+0000 mon.a (mon.0) 984 : cluster 0 mgrmap e22: x(active, starting, since 0.00971573s) 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.595606+0000 mon.c (mon.2) 41 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.595855+0000 mon.c (mon.2) 42 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.596065+0000 mon.c (mon.2) 43 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.596987+0000 mon.c (mon.2) 44 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:35:58.834 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.597299+0000 mon.c (mon.2) 45 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.598518+0000 mon.c (mon.2) 46 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.598942+0000 mon.c (mon.2) 47 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.599257+0000 mon.c (mon.2) 48 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.599514+0000 mon.c (mon.2) 49 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.599805+0000 mon.c (mon.2) 50 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.600104+0000 mon.c (mon.2) 51 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.600409+0000 mon.c (mon.2) 52 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.600655+0000 mon.c (mon.2) 53 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.600917+0000 mon.c (mon.2) 54 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.601873+0000 mon.c (mon.2) 55 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.602131+0000 mon.c (mon.2) 56 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: audit 2024-04-13T07:35:58.602640+0000 mon.c (mon.2) 57 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:35:58.835 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[31522]: cluster 2024-04-13T07:35:58.644205+0000 mon.a (mon.0) 985 : cluster 1 Manager daemon x is now available 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: cluster 2024-04-13T07:35:56.981530+0000 mgr.y (mgr.14373) 241 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.458005+0000 mon.a (mon.0) 974 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.471583+0000 mon.a (mon.0) 975 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.474125+0000 mon.a (mon.0) 976 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.475968+0000 mon.a (mon.0) 977 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.486262+0000 mon.a (mon.0) 978 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.574122+0000 mon.a (mon.0) 979 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.581672+0000 mon.a (mon.0) 980 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-13T07:35:58.836 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: cluster 2024-04-13T07:35:58.582235+0000 mon.a (mon.0) 981 : cluster 1 Activating manager daemon x 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: cluster 2024-04-13T07:35:58.591295+0000 mon.a (mon.0) 982 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.591869+0000 mon.a (mon.0) 983 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: cluster 2024-04-13T07:35:58.591957+0000 mon.a (mon.0) 984 : cluster 0 mgrmap e22: x(active, starting, since 0.00971573s) 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.595606+0000 mon.c (mon.2) 41 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.595855+0000 mon.c (mon.2) 42 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.596065+0000 mon.c (mon.2) 43 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.596987+0000 mon.c (mon.2) 44 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.597299+0000 mon.c (mon.2) 45 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.598518+0000 mon.c (mon.2) 46 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.598942+0000 mon.c (mon.2) 47 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.599257+0000 mon.c (mon.2) 48 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.599514+0000 mon.c (mon.2) 49 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.599805+0000 mon.c (mon.2) 50 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.600104+0000 mon.c (mon.2) 51 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:35:58.837 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.600409+0000 mon.c (mon.2) 52 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:35:58.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.600655+0000 mon.c (mon.2) 53 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:35:58.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.600917+0000 mon.c (mon.2) 54 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:35:58.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.601873+0000 mon.c (mon.2) 55 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:35:58.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.602131+0000 mon.c (mon.2) 56 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:35:58.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: audit 2024-04-13T07:35:58.602640+0000 mon.c (mon.2) 57 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:35:58.838 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:58 smithi089 bash[21353]: cluster 2024-04-13T07:35:58.644205+0000 mon.a (mon.0) 985 : cluster 1 Manager daemon x is now available 2024-04-13T07:35:59.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: [13/Apr/2024:07:35:58] ENGINE Bus STOPPING 2024-04-13T07:35:59.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: [13/Apr/2024:07:35:58] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-13T07:35:59.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: [13/Apr/2024:07:35:58] ENGINE Bus STOPPED 2024-04-13T07:35:59.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: [13/Apr/2024:07:35:58] ENGINE Bus STARTING 2024-04-13T07:35:59.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.767+0000 7f819eaff700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.767+0000 7f819eaff700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.767+0000 7f819eaff700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.767+0000 7f819eaff700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.767+0000 7f819eaff700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.771+0000 7f819b2f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.771+0000 7f819b2f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.771+0000 7f819b2f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.771+0000 7f819b2f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: debug 2024-04-13T07:35:58.771+0000 7f819b2f8700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: [13/Apr/2024:07:35:58] ENGINE Serving on http://:::9283 2024-04-13T07:35:59.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[52286]: [13/Apr/2024:07:35:58] ENGINE Bus STARTED 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: cluster 2024-04-13T07:35:56.981530+0000 mgr.y (mgr.14373) 241 : cluster 0 pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.458005+0000 mon.a (mon.0) 974 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.471583+0000 mon.a (mon.0) 975 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.474125+0000 mon.a (mon.0) 976 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.475968+0000 mon.a (mon.0) 977 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.486262+0000 mon.a (mon.0) 978 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.574122+0000 mon.a (mon.0) 979 : audit 0 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.581672+0000 mon.a (mon.0) 980 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: cluster 2024-04-13T07:35:58.582235+0000 mon.a (mon.0) 981 : cluster 1 Activating manager daemon x 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: cluster 2024-04-13T07:35:58.591295+0000 mon.a (mon.0) 982 : cluster 0 osdmap e90: 8 total, 8 up, 8 in 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.591869+0000 mon.a (mon.0) 983 : audit 1 from='mgr.14373 172.21.15.89:0/94840749' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2024-04-13T07:35:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: cluster 2024-04-13T07:35:58.591957+0000 mon.a (mon.0) 984 : cluster 0 mgrmap e22: x(active, starting, since 0.00971573s) 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.595606+0000 mon.c (mon.2) 41 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.595855+0000 mon.c (mon.2) 42 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.596065+0000 mon.c (mon.2) 43 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.596987+0000 mon.c (mon.2) 44 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.597299+0000 mon.c (mon.2) 45 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.598518+0000 mon.c (mon.2) 46 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.598942+0000 mon.c (mon.2) 47 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.599257+0000 mon.c (mon.2) 48 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.599514+0000 mon.c (mon.2) 49 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.599805+0000 mon.c (mon.2) 50 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.600104+0000 mon.c (mon.2) 51 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.600409+0000 mon.c (mon.2) 52 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:35:59.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.600655+0000 mon.c (mon.2) 53 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:35:59.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.600917+0000 mon.c (mon.2) 54 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:35:59.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.601873+0000 mon.c (mon.2) 55 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:35:59.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.602131+0000 mon.c (mon.2) 56 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:35:59.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: audit 2024-04-13T07:35:58.602640+0000 mon.c (mon.2) 57 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:35:59.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:58 smithi203 bash[23480]: cluster 2024-04-13T07:35:58.644205+0000 mon.a (mon.0) 985 : cluster 1 Manager daemon x is now available 2024-04-13T07:36:00.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:59 smithi203 bash[23480]: audit 2024-04-13T07:35:58.685051+0000 mon.c (mon.2) 58 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:00.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:59 smithi203 bash[23480]: audit 2024-04-13T07:35:58.712641+0000 mon.c (mon.2) 59 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:00.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:59 smithi203 bash[23480]: audit 2024-04-13T07:35:58.720041+0000 mon.c (mon.2) 60 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:00.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:59 smithi203 bash[23480]: audit 2024-04-13T07:35:58.720347+0000 mon.a (mon.0) 986 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:00.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:59 smithi203 bash[23480]: audit 2024-04-13T07:35:58.752836+0000 mon.c (mon.2) 61 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:00.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:59 smithi203 bash[23480]: audit 2024-04-13T07:35:58.753167+0000 mon.a (mon.0) 987 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:00.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:59 smithi203 bash[23480]: audit 2024-04-13T07:35:59.001129+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.89:0/3741461048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:00.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:35:59 smithi203 bash[23480]: cluster 2024-04-13T07:35:59.595739+0000 mon.a (mon.0) 988 : cluster 0 mgrmap e23: x(active, since 1.01348s) 2024-04-13T07:36:00.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[21353]: audit 2024-04-13T07:35:58.685051+0000 mon.c (mon.2) 58 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[21353]: audit 2024-04-13T07:35:58.712641+0000 mon.c (mon.2) 59 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[21353]: audit 2024-04-13T07:35:58.720041+0000 mon.c (mon.2) 60 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[21353]: audit 2024-04-13T07:35:58.720347+0000 mon.a (mon.0) 986 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[21353]: audit 2024-04-13T07:35:58.752836+0000 mon.c (mon.2) 61 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[21353]: audit 2024-04-13T07:35:58.753167+0000 mon.a (mon.0) 987 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[21353]: audit 2024-04-13T07:35:59.001129+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.89:0/3741461048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[21353]: cluster 2024-04-13T07:35:59.595739+0000 mon.a (mon.0) 988 : cluster 0 mgrmap e23: x(active, since 1.01348s) 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[31522]: audit 2024-04-13T07:35:58.685051+0000 mon.c (mon.2) 58 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:00.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[31522]: audit 2024-04-13T07:35:58.712641+0000 mon.c (mon.2) 59 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:00.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[31522]: audit 2024-04-13T07:35:58.720041+0000 mon.c (mon.2) 60 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:00.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[31522]: audit 2024-04-13T07:35:58.720347+0000 mon.a (mon.0) 986 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:00.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[31522]: audit 2024-04-13T07:35:58.752836+0000 mon.c (mon.2) 61 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:00.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[31522]: audit 2024-04-13T07:35:58.753167+0000 mon.a (mon.0) 987 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:00.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[31522]: audit 2024-04-13T07:35:59.001129+0000 mon.c (mon.2) 62 : audit 0 from='client.? 172.21.15.89:0/3741461048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:00.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:35:59 smithi089 bash[31522]: cluster 2024-04-13T07:35:59.595739+0000 mon.a (mon.0) 988 : cluster 0 mgrmap e23: x(active, since 1.01348s) 2024-04-13T07:36:00.678 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21606]: debug 2024-04-13T07:36:00.375+0000 7fe0f406a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:36:00.679 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21606]: debug 2024-04-13T07:36:00.475+0000 7fe0f406a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:36:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:00 smithi203 bash[23480]: cephadm 2024-04-13T07:35:59.157594+0000 mgr.x (mgr.24547) 1 : cephadm 1 Deploying cephadm binary to smithi203 2024-04-13T07:36:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:00 smithi203 bash[23480]: cephadm 2024-04-13T07:35:59.169194+0000 mgr.x (mgr.24547) 2 : cephadm 1 Deploying cephadm binary to smithi089 2024-04-13T07:36:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:00 smithi203 bash[23480]: cluster 2024-04-13T07:35:59.631244+0000 mgr.x (mgr.24547) 3 : cluster 0 pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:00 smithi203 bash[23480]: cephadm 2024-04-13T07:35:59.646763+0000 mgr.x (mgr.24547) 4 : cephadm 1 [13/Apr/2024:07:35:59] ENGINE Bus STARTING 2024-04-13T07:36:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:00 smithi203 bash[23480]: cephadm 2024-04-13T07:35:59.788411+0000 mgr.x (mgr.24547) 5 : cephadm 1 [13/Apr/2024:07:35:59] ENGINE Serving on http://172.21.15.203:8765 2024-04-13T07:36:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:00 smithi203 bash[23480]: cephadm 2024-04-13T07:36:00.003322+0000 mgr.x (mgr.24547) 6 : cephadm 1 [13/Apr/2024:07:36:00] ENGINE Serving on https://172.21.15.203:7150 2024-04-13T07:36:01.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:00 smithi203 bash[23480]: cephadm 2024-04-13T07:36:00.003654+0000 mgr.x (mgr.24547) 7 : cephadm 1 [13/Apr/2024:07:36:00] ENGINE Bus STARTED 2024-04-13T07:36:01.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21606]: debug 2024-04-13T07:36:00.727+0000 7fe0f406a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:36:01.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21353]: cephadm 2024-04-13T07:35:59.157594+0000 mgr.x (mgr.24547) 1 : cephadm 1 Deploying cephadm binary to smithi203 2024-04-13T07:36:01.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21353]: cephadm 2024-04-13T07:35:59.169194+0000 mgr.x (mgr.24547) 2 : cephadm 1 Deploying cephadm binary to smithi089 2024-04-13T07:36:01.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21353]: cluster 2024-04-13T07:35:59.631244+0000 mgr.x (mgr.24547) 3 : cluster 0 pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21353]: cephadm 2024-04-13T07:35:59.646763+0000 mgr.x (mgr.24547) 4 : cephadm 1 [13/Apr/2024:07:35:59] ENGINE Bus STARTING 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21353]: cephadm 2024-04-13T07:35:59.788411+0000 mgr.x (mgr.24547) 5 : cephadm 1 [13/Apr/2024:07:35:59] ENGINE Serving on http://172.21.15.203:8765 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21353]: cephadm 2024-04-13T07:36:00.003322+0000 mgr.x (mgr.24547) 6 : cephadm 1 [13/Apr/2024:07:36:00] ENGINE Serving on https://172.21.15.203:7150 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[21353]: cephadm 2024-04-13T07:36:00.003654+0000 mgr.x (mgr.24547) 7 : cephadm 1 [13/Apr/2024:07:36:00] ENGINE Bus STARTED 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[31522]: cephadm 2024-04-13T07:35:59.157594+0000 mgr.x (mgr.24547) 1 : cephadm 1 Deploying cephadm binary to smithi203 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[31522]: cephadm 2024-04-13T07:35:59.169194+0000 mgr.x (mgr.24547) 2 : cephadm 1 Deploying cephadm binary to smithi089 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[31522]: cluster 2024-04-13T07:35:59.631244+0000 mgr.x (mgr.24547) 3 : cluster 0 pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[31522]: cephadm 2024-04-13T07:35:59.646763+0000 mgr.x (mgr.24547) 4 : cephadm 1 [13/Apr/2024:07:35:59] ENGINE Bus STARTING 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[31522]: cephadm 2024-04-13T07:35:59.788411+0000 mgr.x (mgr.24547) 5 : cephadm 1 [13/Apr/2024:07:35:59] ENGINE Serving on http://172.21.15.203:8765 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[31522]: cephadm 2024-04-13T07:36:00.003322+0000 mgr.x (mgr.24547) 6 : cephadm 1 [13/Apr/2024:07:36:00] ENGINE Serving on https://172.21.15.203:7150 2024-04-13T07:36:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:00 smithi089 bash[31522]: cephadm 2024-04-13T07:36:00.003654+0000 mgr.x (mgr.24547) 7 : cephadm 1 [13/Apr/2024:07:36:00] ENGINE Bus STARTED 2024-04-13T07:36:01.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:01 smithi089 bash[31522]: cluster 2024-04-13T07:36:00.605742+0000 mgr.x (mgr.24547) 8 : cluster 0 pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:01.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:01 smithi089 bash[31522]: audit 2024-04-13T07:36:01.440043+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.89:0/555271527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:01.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:01 smithi089 bash[21606]: debug 2024-04-13T07:36:01.527+0000 7fe0f406a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:36:01.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:01 smithi089 bash[21606]: debug 2024-04-13T07:36:01.639+0000 7fe0f406a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:36:01.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:01 smithi089 bash[21606]: debug 2024-04-13T07:36:01.751+0000 7fe0f406a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:36:01.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:01 smithi089 bash[21353]: cluster 2024-04-13T07:36:00.605742+0000 mgr.x (mgr.24547) 8 : cluster 0 pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:01.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:01 smithi089 bash[21353]: audit 2024-04-13T07:36:01.440043+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.89:0/555271527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:02.011 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:01 smithi203 bash[23480]: cluster 2024-04-13T07:36:00.605742+0000 mgr.x (mgr.24547) 8 : cluster 0 pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:02.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:01 smithi203 bash[23480]: audit 2024-04-13T07:36:01.440043+0000 mon.c (mon.2) 63 : audit 0 from='client.? 172.21.15.89:0/555271527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:02.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:02 smithi089 bash[21606]: debug 2024-04-13T07:36:02.107+0000 7fe0f406a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:36:02.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:02 smithi089 bash[21606]: debug 2024-04-13T07:36:02.219+0000 7fe0f406a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:36:02.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:02 smithi089 bash[21606]: debug 2024-04-13T07:36:02.323+0000 7fe0f406a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:36:03.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:02 smithi203 bash[23480]: cluster 2024-04-13T07:36:01.695676+0000 mon.a (mon.0) 989 : cluster 0 mgrmap e24: x(active, since 3s) 2024-04-13T07:36:03.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:02 smithi089 bash[31522]: cluster 2024-04-13T07:36:01.695676+0000 mon.a (mon.0) 989 : cluster 0 mgrmap e24: x(active, since 3s) 2024-04-13T07:36:03.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:02 smithi089 bash[21353]: cluster 2024-04-13T07:36:01.695676+0000 mon.a (mon.0) 989 : cluster 0 mgrmap e24: x(active, since 3s) 2024-04-13T07:36:03.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:02 smithi089 bash[21606]: debug 2024-04-13T07:36:02.683+0000 7fe0f406a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:36:03.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:02 smithi089 bash[21606]: debug 2024-04-13T07:36:02.931+0000 7fe0f406a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:36:03.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:03 smithi089 bash[21606]: debug 2024-04-13T07:36:03.203+0000 7fe0f406a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:36:03.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:03 smithi089 bash[21606]: debug 2024-04-13T07:36:03.319+0000 7fe0f406a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:36:03.857 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:03 smithi089 bash[21606]: debug 2024-04-13T07:36:03.607+0000 7fe0f406a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:36:04.261 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:03 smithi203 bash[23480]: cluster 2024-04-13T07:36:02.606300+0000 mgr.x (mgr.24547) 9 : cluster 0 pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:04.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:03 smithi203 bash[23480]: audit 2024-04-13T07:36:03.877323+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.89:0/441685931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:04.274 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:03 smithi089 bash[21353]: cluster 2024-04-13T07:36:02.606300+0000 mgr.x (mgr.24547) 9 : cluster 0 pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:04.274 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:03 smithi089 bash[21353]: audit 2024-04-13T07:36:03.877323+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.89:0/441685931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:04.274 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:03 smithi089 bash[21606]: debug 2024-04-13T07:36:03.859+0000 7fe0f406a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:36:04.275 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:03 smithi089 bash[31522]: cluster 2024-04-13T07:36:02.606300+0000 mgr.x (mgr.24547) 9 : cluster 0 pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:04.275 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:03 smithi089 bash[31522]: audit 2024-04-13T07:36:03.877323+0000 mon.a (mon.0) 990 : audit 0 from='client.? 172.21.15.89:0/441685931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:04.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:04 smithi089 bash[21606]: debug 2024-04-13T07:36:04.275+0000 7fe0f406a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:36:04.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:04 smithi089 bash[21606]: debug 2024-04-13T07:36:04.395+0000 7fe0f406a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:36:05.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:04 smithi089 bash[21606]: debug 2024-04-13T07:36:04.651+0000 7fe0f406a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:36:05.928 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:05 smithi089 bash[21606]: debug 2024-04-13T07:36:05.667+0000 7fe0f406a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:36:05.928 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:05 smithi089 bash[21606]: debug 2024-04-13T07:36:05.787+0000 7fe0f406a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:36:06.225 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:05 smithi089 bash[21353]: cluster 2024-04-13T07:36:04.607065+0000 mgr.x (mgr.24547) 10 : cluster 0 pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:06.225 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:05 smithi089 bash[31522]: cluster 2024-04-13T07:36:04.607065+0000 mgr.x (mgr.24547) 10 : cluster 0 pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:06.225 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: debug 2024-04-13T07:36:06.103+0000 7fe0f406a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:36:06.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:05 smithi203 bash[23480]: cluster 2024-04-13T07:36:04.607065+0000 mgr.x (mgr.24547) 10 : cluster 0 pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:06.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: debug 2024-04-13T07:36:06.227+0000 7fe0f406a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:36:06.952 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: debug 2024-04-13T07:36:06.699+0000 7fe0f406a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:36:06.952 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: debug 2024-04-13T07:36:06.827+0000 7fe0f406a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:36:06.953 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: [13/Apr/2024:07:36:06] ENGINE Bus STARTING 2024-04-13T07:36:06.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[31522]: audit 2024-04-13T07:36:06.308173+0000 mon.a (mon.0) 991 : audit 0 from='client.? 172.21.15.89:0/629055120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:06.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[31522]: cluster 2024-04-13T07:36:06.852048+0000 mon.a (mon.0) 992 : cluster 0 Standby manager daemon y started 2024-04-13T07:36:06.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[31522]: audit 2024-04-13T07:36:06.855142+0000 mon.a (mon.0) 993 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-13T07:36:06.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[31522]: audit 2024-04-13T07:36:06.855667+0000 mon.a (mon.0) 994 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:36:06.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[31522]: audit 2024-04-13T07:36:06.857380+0000 mon.a (mon.0) 995 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-13T07:36:06.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[31522]: audit 2024-04-13T07:36:06.858395+0000 mon.a (mon.0) 996 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:36:06.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21353]: audit 2024-04-13T07:36:06.308173+0000 mon.a (mon.0) 991 : audit 0 from='client.? 172.21.15.89:0/629055120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:06.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21353]: cluster 2024-04-13T07:36:06.852048+0000 mon.a (mon.0) 992 : cluster 0 Standby manager daemon y started 2024-04-13T07:36:06.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21353]: audit 2024-04-13T07:36:06.855142+0000 mon.a (mon.0) 993 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-13T07:36:06.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21353]: audit 2024-04-13T07:36:06.855667+0000 mon.a (mon.0) 994 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:36:06.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21353]: audit 2024-04-13T07:36:06.857380+0000 mon.a (mon.0) 995 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-13T07:36:06.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21353]: audit 2024-04-13T07:36:06.858395+0000 mon.a (mon.0) 996 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:36:07.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:06 smithi203 bash[23480]: audit 2024-04-13T07:36:06.308173+0000 mon.a (mon.0) 991 : audit 0 from='client.? 172.21.15.89:0/629055120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:07.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:06 smithi203 bash[23480]: cluster 2024-04-13T07:36:06.852048+0000 mon.a (mon.0) 992 : cluster 0 Standby manager daemon y started 2024-04-13T07:36:07.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:06 smithi203 bash[23480]: audit 2024-04-13T07:36:06.855142+0000 mon.a (mon.0) 993 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-13T07:36:07.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:06 smithi203 bash[23480]: audit 2024-04-13T07:36:06.855667+0000 mon.a (mon.0) 994 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:36:07.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:06 smithi203 bash[23480]: audit 2024-04-13T07:36:06.857380+0000 mon.a (mon.0) 995 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-13T07:36:07.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:06 smithi203 bash[23480]: audit 2024-04-13T07:36:06.858395+0000 mon.a (mon.0) 996 : audit 0 from='mgr.? 172.21.15.89:0/3734370634' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:36:07.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: CherryPy Checker: 2024-04-13T07:36:07.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: The Application mounted at '' has an empty config. 2024-04-13T07:36:07.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: [13/Apr/2024:07:36:06] ENGINE Serving on http://:::9283 2024-04-13T07:36:07.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:06 smithi089 bash[21606]: [13/Apr/2024:07:36:06] ENGINE Bus STARTED 2024-04-13T07:36:08.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:07 smithi203 bash[23480]: cluster 2024-04-13T07:36:06.608011+0000 mgr.x (mgr.24547) 11 : cluster 0 pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:36:08.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:07 smithi203 bash[23480]: cluster 2024-04-13T07:36:06.948709+0000 mon.a (mon.0) 997 : cluster 0 mgrmap e25: x(active, since 8s), standbys: y 2024-04-13T07:36:08.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:07 smithi203 bash[23480]: audit 2024-04-13T07:36:06.953860+0000 mon.c (mon.2) 64 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:36:08.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:07 smithi089 bash[31522]: cluster 2024-04-13T07:36:06.608011+0000 mgr.x (mgr.24547) 11 : cluster 0 pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:36:08.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:07 smithi089 bash[31522]: cluster 2024-04-13T07:36:06.948709+0000 mon.a (mon.0) 997 : cluster 0 mgrmap e25: x(active, since 8s), standbys: y 2024-04-13T07:36:08.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:07 smithi089 bash[31522]: audit 2024-04-13T07:36:06.953860+0000 mon.c (mon.2) 64 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:36:08.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:07 smithi089 bash[21353]: cluster 2024-04-13T07:36:06.608011+0000 mgr.x (mgr.24547) 11 : cluster 0 pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:36:08.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:07 smithi089 bash[21353]: cluster 2024-04-13T07:36:06.948709+0000 mon.a (mon.0) 997 : cluster 0 mgrmap e25: x(active, since 8s), standbys: y 2024-04-13T07:36:08.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:07 smithi089 bash[21353]: audit 2024-04-13T07:36:06.953860+0000 mon.c (mon.2) 64 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:36:09.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:08 smithi203 bash[23480]: audit 2024-04-13T07:36:08.743567+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.89:0/3182382423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:09.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:08 smithi089 bash[21353]: audit 2024-04-13T07:36:08.743567+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.89:0/3182382423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:09.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:08 smithi089 bash[31522]: audit 2024-04-13T07:36:08.743567+0000 mon.c (mon.2) 65 : audit 0 from='client.? 172.21.15.89:0/3182382423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:10.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:09 smithi203 bash[23480]: cluster 2024-04-13T07:36:08.608693+0000 mgr.x (mgr.24547) 12 : cluster 0 pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:36:10.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:09 smithi089 bash[21353]: cluster 2024-04-13T07:36:08.608693+0000 mgr.x (mgr.24547) 12 : cluster 0 pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:36:10.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:09 smithi089 bash[31522]: cluster 2024-04-13T07:36:08.608693+0000 mgr.x (mgr.24547) 12 : cluster 0 pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:36:12.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:11 smithi203 bash[23480]: cluster 2024-04-13T07:36:10.609894+0000 mgr.x (mgr.24547) 13 : cluster 0 pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:36:12.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:11 smithi203 bash[23480]: audit 2024-04-13T07:36:11.189304+0000 mon.a (mon.0) 998 : audit 0 from='client.? 172.21.15.89:0/2605358428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:12.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:11 smithi089 bash[31522]: cluster 2024-04-13T07:36:10.609894+0000 mgr.x (mgr.24547) 13 : cluster 0 pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:36:12.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:11 smithi089 bash[31522]: audit 2024-04-13T07:36:11.189304+0000 mon.a (mon.0) 998 : audit 0 from='client.? 172.21.15.89:0/2605358428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:11 smithi089 bash[21353]: cluster 2024-04-13T07:36:10.609894+0000 mgr.x (mgr.24547) 13 : cluster 0 pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:36:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:11 smithi089 bash[21353]: audit 2024-04-13T07:36:11.189304+0000 mon.a (mon.0) 998 : audit 0 from='client.? 172.21.15.89:0/2605358428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:13.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:12 smithi203 bash[23480]: audit 2024-04-13T07:36:11.980667+0000 mon.a (mon.0) 999 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:12 smithi203 bash[23480]: audit 2024-04-13T07:36:11.991056+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:12 smithi203 bash[23480]: audit 2024-04-13T07:36:12.367638+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:12 smithi203 bash[23480]: audit 2024-04-13T07:36:12.380634+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:12 smithi089 bash[21353]: audit 2024-04-13T07:36:11.980667+0000 mon.a (mon.0) 999 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:12 smithi089 bash[21353]: audit 2024-04-13T07:36:11.991056+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:12 smithi089 bash[21353]: audit 2024-04-13T07:36:12.367638+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:12 smithi089 bash[21353]: audit 2024-04-13T07:36:12.380634+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:12 smithi089 bash[31522]: audit 2024-04-13T07:36:11.980667+0000 mon.a (mon.0) 999 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:12 smithi089 bash[31522]: audit 2024-04-13T07:36:11.991056+0000 mon.a (mon.0) 1000 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:12 smithi089 bash[31522]: audit 2024-04-13T07:36:12.367638+0000 mon.a (mon.0) 1001 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:13.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:12 smithi089 bash[31522]: audit 2024-04-13T07:36:12.380634+0000 mon.a (mon.0) 1002 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: cluster 2024-04-13T07:36:12.610436+0000 mgr.x (mgr.24547) 14 : cluster 0 pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.112991+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.130492+0000 mon.a (mon.0) 1004 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.135192+0000 mon.c (mon.2) 66 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.135895+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.138246+0000 mon.c (mon.2) 67 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.138949+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.141086+0000 mon.c (mon.2) 68 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.141780+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.143852+0000 mon.c (mon.2) 69 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.144517+0000 mon.a (mon.0) 1008 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.157957+0000 mon.a (mon.0) 1009 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.443687+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.461159+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.612844+0000 mon.a (mon.0) 1012 : audit 0 from='client.? 172.21.15.89:0/2384741977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:14.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:13 smithi203 bash[23480]: audit 2024-04-13T07:36:13.712190+0000 mon.c (mon.2) 70 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: cluster 2024-04-13T07:36:12.610436+0000 mgr.x (mgr.24547) 14 : cluster 0 pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.112991+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.130492+0000 mon.a (mon.0) 1004 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.135192+0000 mon.c (mon.2) 66 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.135895+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.138246+0000 mon.c (mon.2) 67 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.138949+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.141086+0000 mon.c (mon.2) 68 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.141780+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.143852+0000 mon.c (mon.2) 69 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.144517+0000 mon.a (mon.0) 1008 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.157957+0000 mon.a (mon.0) 1009 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.443687+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.461159+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.612844+0000 mon.a (mon.0) 1012 : audit 0 from='client.? 172.21.15.89:0/2384741977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[21353]: audit 2024-04-13T07:36:13.712190+0000 mon.c (mon.2) 70 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: cluster 2024-04-13T07:36:12.610436+0000 mgr.x (mgr.24547) 14 : cluster 0 pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.112991+0000 mon.a (mon.0) 1003 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.130492+0000 mon.a (mon.0) 1004 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.135192+0000 mon.c (mon.2) 66 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.135895+0000 mon.a (mon.0) 1005 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.138246+0000 mon.c (mon.2) 67 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.138949+0000 mon.a (mon.0) 1006 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.141086+0000 mon.c (mon.2) 68 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.141780+0000 mon.a (mon.0) 1007 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.143852+0000 mon.c (mon.2) 69 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.144517+0000 mon.a (mon.0) 1008 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.157957+0000 mon.a (mon.0) 1009 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.443687+0000 mon.a (mon.0) 1010 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.461159+0000 mon.a (mon.0) 1011 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.612844+0000 mon.a (mon.0) 1012 : audit 0 from='client.? 172.21.15.89:0/2384741977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:14.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:13 smithi089 bash[31522]: audit 2024-04-13T07:36:13.712190+0000 mon.c (mon.2) 70 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:15.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:14 smithi203 bash[23480]: cephadm 2024-04-13T07:36:13.149759+0000 mgr.x (mgr.24547) 15 : cephadm 1 Adjusting osd_memory_target on smithi089 to 2524M 2024-04-13T07:36:15.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:14 smithi089 bash[21353]: cephadm 2024-04-13T07:36:13.149759+0000 mgr.x (mgr.24547) 15 : cephadm 1 Adjusting osd_memory_target on smithi089 to 2524M 2024-04-13T07:36:15.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:14 smithi089 bash[31522]: cephadm 2024-04-13T07:36:13.149759+0000 mgr.x (mgr.24547) 15 : cephadm 1 Adjusting osd_memory_target on smithi089 to 2524M 2024-04-13T07:36:16.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:15 smithi203 bash[23480]: cluster 2024-04-13T07:36:14.611311+0000 mgr.x (mgr.24547) 16 : cluster 0 pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:16.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:15 smithi089 bash[21353]: cluster 2024-04-13T07:36:14.611311+0000 mgr.x (mgr.24547) 16 : cluster 0 pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:16.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:16 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:36:16] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-13T07:36:16.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:15 smithi089 bash[31522]: cluster 2024-04-13T07:36:14.611311+0000 mgr.x (mgr.24547) 16 : cluster 0 pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:17.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:17 smithi203 bash[23480]: audit 2024-04-13T07:36:16.030825+0000 mon.a (mon.0) 1013 : audit 0 from='client.? 172.21.15.89:0/1258090626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:17.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:17 smithi089 bash[31522]: audit 2024-04-13T07:36:16.030825+0000 mon.a (mon.0) 1013 : audit 0 from='client.? 172.21.15.89:0/1258090626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:17.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:17 smithi089 bash[21353]: audit 2024-04-13T07:36:16.030825+0000 mon.a (mon.0) 1013 : audit 0 from='client.? 172.21.15.89:0/1258090626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:18.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:18 smithi203 bash[23480]: cluster 2024-04-13T07:36:16.612284+0000 mgr.x (mgr.24547) 17 : cluster 0 pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:18.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:18 smithi089 bash[31522]: cluster 2024-04-13T07:36:16.612284+0000 mgr.x (mgr.24547) 17 : cluster 0 pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:18.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:18 smithi089 bash[21353]: cluster 2024-04-13T07:36:16.612284+0000 mgr.x (mgr.24547) 17 : cluster 0 pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:36:19.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:19 smithi203 bash[23480]: audit 2024-04-13T07:36:18.464406+0000 mon.a (mon.0) 1014 : audit 0 from='client.? 172.21.15.89:0/1977491311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:19.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:19 smithi089 bash[21353]: audit 2024-04-13T07:36:18.464406+0000 mon.a (mon.0) 1014 : audit 0 from='client.? 172.21.15.89:0/1977491311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:19.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:19 smithi089 bash[31522]: audit 2024-04-13T07:36:18.464406+0000 mon.a (mon.0) 1014 : audit 0 from='client.? 172.21.15.89:0/1977491311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:20.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:20 smithi203 bash[23480]: cluster 2024-04-13T07:36:18.612994+0000 mgr.x (mgr.24547) 18 : cluster 0 pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:20.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:20 smithi089 bash[21353]: cluster 2024-04-13T07:36:18.612994+0000 mgr.x (mgr.24547) 18 : cluster 0 pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:20.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:20 smithi089 bash[31522]: cluster 2024-04-13T07:36:18.612994+0000 mgr.x (mgr.24547) 18 : cluster 0 pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:21.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:21 smithi089 bash[21353]: cluster 2024-04-13T07:36:20.614113+0000 mgr.x (mgr.24547) 19 : cluster 0 pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:21.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:21 smithi089 bash[21353]: audit 2024-04-13T07:36:20.905025+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.89:0/1659867589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:21.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:21 smithi089 bash[31522]: cluster 2024-04-13T07:36:20.614113+0000 mgr.x (mgr.24547) 19 : cluster 0 pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:21.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:21 smithi089 bash[31522]: audit 2024-04-13T07:36:20.905025+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.89:0/1659867589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:21.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:21 smithi203 bash[23480]: cluster 2024-04-13T07:36:20.614113+0000 mgr.x (mgr.24547) 19 : cluster 0 pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:21.488 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:21 smithi203 bash[23480]: audit 2024-04-13T07:36:20.905025+0000 mon.c (mon.2) 71 : audit 0 from='client.? 172.21.15.89:0/1659867589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:24.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: cluster 2024-04-13T07:36:22.614648+0000 mgr.x (mgr.24547) 20 : cluster 0 pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.346685+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.89:0/2887392579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.517654+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.532220+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.536334+0000 mon.c (mon.2) 73 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.537051+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.539352+0000 mon.c (mon.2) 74 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.539947+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.542199+0000 mon.c (mon.2) 75 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.542810+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.544856+0000 mon.c (mon.2) 76 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.545525+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.555682+0000 mon.a (mon.0) 1021 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.572028+0000 mon.c (mon.2) 77 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:24.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:23 smithi203 bash[23480]: audit 2024-04-13T07:36:23.573955+0000 mon.c (mon.2) 78 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: cluster 2024-04-13T07:36:22.614648+0000 mgr.x (mgr.24547) 20 : cluster 0 pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.346685+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.89:0/2887392579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.517654+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.532220+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.536334+0000 mon.c (mon.2) 73 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.537051+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.539352+0000 mon.c (mon.2) 74 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.539947+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.542199+0000 mon.c (mon.2) 75 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.542810+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.544856+0000 mon.c (mon.2) 76 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.545525+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.555682+0000 mon.a (mon.0) 1021 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.572028+0000 mon.c (mon.2) 77 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:24.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[21353]: audit 2024-04-13T07:36:23.573955+0000 mon.c (mon.2) 78 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:36:24.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: cluster 2024-04-13T07:36:22.614648+0000 mgr.x (mgr.24547) 20 : cluster 0 pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:24.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.346685+0000 mon.c (mon.2) 72 : audit 0 from='client.? 172.21.15.89:0/2887392579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:24.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.517654+0000 mon.a (mon.0) 1015 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.532220+0000 mon.a (mon.0) 1016 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.536334+0000 mon.c (mon.2) 73 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.537051+0000 mon.a (mon.0) 1017 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.539352+0000 mon.c (mon.2) 74 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.539947+0000 mon.a (mon.0) 1018 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.542199+0000 mon.c (mon.2) 75 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.542810+0000 mon.a (mon.0) 1019 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.544856+0000 mon.c (mon.2) 76 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.545525+0000 mon.a (mon.0) 1020 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2024-04-13T07:36:24.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.555682+0000 mon.a (mon.0) 1021 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:24.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.572028+0000 mon.c (mon.2) 77 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:24.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:23 smithi089 bash[31522]: audit 2024-04-13T07:36:23.573955+0000 mon.c (mon.2) 78 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:36:25.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:24 smithi203 bash[23480]: cephadm 2024-04-13T07:36:23.550526+0000 mgr.x (mgr.24547) 21 : cephadm 1 Adjusting osd_memory_target on smithi203 to 2524M 2024-04-13T07:36:25.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:24 smithi203 bash[23480]: cephadm 2024-04-13T07:36:23.580055+0000 mgr.x (mgr.24547) 22 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:36:25.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:24 smithi203 bash[23480]: cephadm 2024-04-13T07:36:23.580813+0000 mgr.x (mgr.24547) 23 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:36:25.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:24 smithi203 bash[23480]: audit 2024-04-13T07:36:24.628444+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:24 smithi203 bash[23480]: audit 2024-04-13T07:36:24.633985+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:24 smithi203 bash[23480]: audit 2024-04-13T07:36:24.644378+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:24 smithi203 bash[23480]: audit 2024-04-13T07:36:24.649539+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:24 smithi203 bash[23480]: audit 2024-04-13T07:36:24.660704+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[21353]: cephadm 2024-04-13T07:36:23.550526+0000 mgr.x (mgr.24547) 21 : cephadm 1 Adjusting osd_memory_target on smithi203 to 2524M 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[21353]: cephadm 2024-04-13T07:36:23.580055+0000 mgr.x (mgr.24547) 22 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[21353]: cephadm 2024-04-13T07:36:23.580813+0000 mgr.x (mgr.24547) 23 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[21353]: audit 2024-04-13T07:36:24.628444+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[21353]: audit 2024-04-13T07:36:24.633985+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[21353]: audit 2024-04-13T07:36:24.644378+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[21353]: audit 2024-04-13T07:36:24.649539+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[21353]: audit 2024-04-13T07:36:24.660704+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[31522]: cephadm 2024-04-13T07:36:23.550526+0000 mgr.x (mgr.24547) 21 : cephadm 1 Adjusting osd_memory_target on smithi203 to 2524M 2024-04-13T07:36:25.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[31522]: cephadm 2024-04-13T07:36:23.580055+0000 mgr.x (mgr.24547) 22 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:36:25.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[31522]: cephadm 2024-04-13T07:36:23.580813+0000 mgr.x (mgr.24547) 23 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:36:25.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[31522]: audit 2024-04-13T07:36:24.628444+0000 mon.a (mon.0) 1022 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[31522]: audit 2024-04-13T07:36:24.633985+0000 mon.a (mon.0) 1023 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[31522]: audit 2024-04-13T07:36:24.644378+0000 mon.a (mon.0) 1024 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[31522]: audit 2024-04-13T07:36:24.649539+0000 mon.a (mon.0) 1025 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:25.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:24 smithi089 bash[31522]: audit 2024-04-13T07:36:24.660704+0000 mon.a (mon.0) 1026 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:26.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:25 smithi203 bash[23480]: cephadm 2024-04-13T07:36:24.130570+0000 mgr.x (mgr.24547) 24 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:36:26.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:25 smithi203 bash[23480]: cephadm 2024-04-13T07:36:24.130943+0000 mgr.x (mgr.24547) 25 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:36:26.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:25 smithi203 bash[23480]: cephadm 2024-04-13T07:36:24.295121+0000 mgr.x (mgr.24547) 26 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:36:26.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:25 smithi203 bash[23480]: cephadm 2024-04-13T07:36:24.296074+0000 mgr.x (mgr.24547) 27 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:36:26.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:25 smithi203 bash[23480]: cephadm 2024-04-13T07:36:24.459761+0000 mgr.x (mgr.24547) 28 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:36:26.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:25 smithi203 bash[23480]: cephadm 2024-04-13T07:36:24.460624+0000 mgr.x (mgr.24547) 29 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:36:26.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:25 smithi203 bash[23480]: cluster 2024-04-13T07:36:24.615215+0000 mgr.x (mgr.24547) 30 : cluster 0 pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:26.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:25 smithi203 bash[23480]: cephadm 2024-04-13T07:36:24.742309+0000 mgr.x (mgr.24547) 31 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:36:26.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[21353]: cephadm 2024-04-13T07:36:24.130570+0000 mgr.x (mgr.24547) 24 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:36:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[21353]: cephadm 2024-04-13T07:36:24.130943+0000 mgr.x (mgr.24547) 25 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:36:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[21353]: cephadm 2024-04-13T07:36:24.295121+0000 mgr.x (mgr.24547) 26 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:36:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[21353]: cephadm 2024-04-13T07:36:24.296074+0000 mgr.x (mgr.24547) 27 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:36:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[21353]: cephadm 2024-04-13T07:36:24.459761+0000 mgr.x (mgr.24547) 28 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:36:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[21353]: cephadm 2024-04-13T07:36:24.460624+0000 mgr.x (mgr.24547) 29 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:36:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[21353]: cluster 2024-04-13T07:36:24.615215+0000 mgr.x (mgr.24547) 30 : cluster 0 pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[21353]: cephadm 2024-04-13T07:36:24.742309+0000 mgr.x (mgr.24547) 31 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:36:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[31522]: cephadm 2024-04-13T07:36:24.130570+0000 mgr.x (mgr.24547) 24 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:36:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[31522]: cephadm 2024-04-13T07:36:24.130943+0000 mgr.x (mgr.24547) 25 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:36:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[31522]: cephadm 2024-04-13T07:36:24.295121+0000 mgr.x (mgr.24547) 26 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:36:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[31522]: cephadm 2024-04-13T07:36:24.296074+0000 mgr.x (mgr.24547) 27 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:36:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[31522]: cephadm 2024-04-13T07:36:24.459761+0000 mgr.x (mgr.24547) 28 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:36:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[31522]: cephadm 2024-04-13T07:36:24.460624+0000 mgr.x (mgr.24547) 29 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:36:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[31522]: cluster 2024-04-13T07:36:24.615215+0000 mgr.x (mgr.24547) 30 : cluster 0 pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:26.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:25 smithi089 bash[31522]: cephadm 2024-04-13T07:36:24.742309+0000 mgr.x (mgr.24547) 31 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:36:26.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:26 smithi089 bash[21606]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:36:26] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-13T07:36:27.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:26 smithi203 bash[23480]: cephadm 2024-04-13T07:36:25.032165+0000 mgr.x (mgr.24547) 32 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:36:27.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:26 smithi203 bash[23480]: audit 2024-04-13T07:36:25.790755+0000 mon.a (mon.0) 1027 : audit 0 from='client.? 172.21.15.89:0/2035291624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:27.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:26 smithi089 bash[31522]: cephadm 2024-04-13T07:36:25.032165+0000 mgr.x (mgr.24547) 32 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:36:27.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:26 smithi089 bash[31522]: audit 2024-04-13T07:36:25.790755+0000 mon.a (mon.0) 1027 : audit 0 from='client.? 172.21.15.89:0/2035291624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:27.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:26 smithi089 bash[21353]: cephadm 2024-04-13T07:36:25.032165+0000 mgr.x (mgr.24547) 32 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:36:27.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:26 smithi089 bash[21353]: audit 2024-04-13T07:36:25.790755+0000 mon.a (mon.0) 1027 : audit 0 from='client.? 172.21.15.89:0/2035291624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:28.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:27 smithi203 bash[23480]: cluster 2024-04-13T07:36:26.615907+0000 mgr.x (mgr.24547) 33 : cluster 0 pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:28.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:27 smithi089 bash[21353]: cluster 2024-04-13T07:36:26.615907+0000 mgr.x (mgr.24547) 33 : cluster 0 pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:28.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:27 smithi089 bash[31522]: cluster 2024-04-13T07:36:26.615907+0000 mgr.x (mgr.24547) 33 : cluster 0 pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:28.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:28 smithi203 bash[23480]: audit 2024-04-13T07:36:28.224720+0000 mon.a (mon.0) 1028 : audit 0 from='client.? 172.21.15.89:0/160901324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:29.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:28 smithi089 bash[21353]: audit 2024-04-13T07:36:28.224720+0000 mon.a (mon.0) 1028 : audit 0 from='client.? 172.21.15.89:0/160901324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:29.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:28 smithi089 bash[31522]: audit 2024-04-13T07:36:28.224720+0000 mon.a (mon.0) 1028 : audit 0 from='client.? 172.21.15.89:0/160901324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:29.696 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 systemd[1]: Stopping Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:36:29.696 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.519Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-13T07:36:29.696 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.519Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.519Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.519Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.519Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.519Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.519Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.519Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.520Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.520Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.520Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-13T07:36:29.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[50456]: ts=2024-04-13T07:36:29.520Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-13T07:36:30.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[23480]: cluster 2024-04-13T07:36:28.616307+0000 mgr.x (mgr.24547) 34 : cluster 0 pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:30.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:29 smithi203 bash[23480]: audit 2024-04-13T07:36:28.711994+0000 mon.c (mon.2) 79 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:30.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:29 smithi089 bash[21353]: cluster 2024-04-13T07:36:28.616307+0000 mgr.x (mgr.24547) 34 : cluster 0 pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:30.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:29 smithi089 bash[21353]: audit 2024-04-13T07:36:28.711994+0000 mon.c (mon.2) 79 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:29 smithi089 bash[31522]: cluster 2024-04-13T07:36:28.616307+0000 mgr.x (mgr.24547) 34 : cluster 0 pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:29 smithi089 bash[31522]: audit 2024-04-13T07:36:28.711994+0000 mon.c (mon.2) 79 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:30.444 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:30 smithi203 bash[58828]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-prometheus-a 2024-04-13T07:36:30.445 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:30 smithi203 bash[58890]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-prometheus-a 2024-04-13T07:36:30.445 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:30 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-13T07:36:30.445 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:30 smithi203 systemd[1]: Stopped Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:36:30.445 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:30 smithi203 systemd[1]: Started Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:36:31.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: audit 2024-04-13T07:36:30.457784+0000 mon.a (mon.0) 1029 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:31.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: audit 2024-04-13T07:36:30.475285+0000 mon.a (mon.0) 1030 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:31.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: audit 2024-04-13T07:36:30.520758+0000 mon.c (mon.2) 80 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:36:31.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: audit 2024-04-13T07:36:30.525478+0000 mgr.x (mgr.24547) 35 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:36:31.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: audit 2024-04-13T07:36:30.537229+0000 mon.c (mon.2) 81 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:31.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: cephadm 2024-04-13T07:36:30.543236+0000 mgr.x (mgr.24547) 36 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-04-13T07:36:31.725 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: cephadm 2024-04-13T07:36:30.544885+0000 mgr.x (mgr.24547) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: cluster 2024-04-13T07:36:30.617327+0000 mgr.x (mgr.24547) 38 : cluster 0 pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[21353]: audit 2024-04-13T07:36:30.655751+0000 mon.a (mon.0) 1031 : audit 0 from='client.? 172.21.15.89:0/3660562442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: audit 2024-04-13T07:36:30.457784+0000 mon.a (mon.0) 1029 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: audit 2024-04-13T07:36:30.475285+0000 mon.a (mon.0) 1030 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: audit 2024-04-13T07:36:30.520758+0000 mon.c (mon.2) 80 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: audit 2024-04-13T07:36:30.525478+0000 mgr.x (mgr.24547) 35 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: audit 2024-04-13T07:36:30.537229+0000 mon.c (mon.2) 81 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: cephadm 2024-04-13T07:36:30.543236+0000 mgr.x (mgr.24547) 36 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: cephadm 2024-04-13T07:36:30.544885+0000 mgr.x (mgr.24547) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: cluster 2024-04-13T07:36:30.617327+0000 mgr.x (mgr.24547) 38 : cluster 0 pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:31.726 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:31 smithi089 bash[31522]: audit 2024-04-13T07:36:30.655751+0000 mon.a (mon.0) 1031 : audit 0 from='client.? 172.21.15.89:0/3660562442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:31.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: audit 2024-04-13T07:36:30.457784+0000 mon.a (mon.0) 1029 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:31.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: audit 2024-04-13T07:36:30.475285+0000 mon.a (mon.0) 1030 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:31.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: audit 2024-04-13T07:36:30.520758+0000 mon.c (mon.2) 80 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:36:31.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: audit 2024-04-13T07:36:30.525478+0000 mgr.x (mgr.24547) 35 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:36:31.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: audit 2024-04-13T07:36:30.537229+0000 mon.c (mon.2) 81 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:31.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: cephadm 2024-04-13T07:36:30.543236+0000 mgr.x (mgr.24547) 36 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-04-13T07:36:31.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: cephadm 2024-04-13T07:36:30.544885+0000 mgr.x (mgr.24547) 37 : cephadm 1 Upgrade: Need to upgrade myself (mgr.x) 2024-04-13T07:36:31.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: cluster 2024-04-13T07:36:30.617327+0000 mgr.x (mgr.24547) 38 : cluster 0 pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:31.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[23480]: audit 2024-04-13T07:36:30.655751+0000 mon.a (mon.0) 1031 : audit 0 from='client.? 172.21.15.89:0/3660562442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:31.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.560Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-13T07:36:31.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.560Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-13T07:36:31.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.560Z caller=main.go:570 level=info host_details="(Linux 5.15.0-102-generic #112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024 x86_64 smithi203 (none))" 2024-04-13T07:36:31.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.561Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-13T07:36:31.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.561Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-13T07:36:31.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.562Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-13T07:36:31.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.563Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-13T07:36:31.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.565Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.565Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.567Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.567Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.693µs 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.567Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.568Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.579Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.580Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.580Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=41.456µs wal_replay_duration=13.255068ms wbl_replay_duration=136ns total_replay_duration=13.330046ms 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.582Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.582Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.582Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.607Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=25.047446ms db_storage=970ns remote_storage=1.716µs web_handler=434ns query_engine=826ns scrape=259.253µs scrape_sd=107.783µs notify=10.874µs notify_sd=16.124µs rules=24.1387ms tracing=6.269µs 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.607Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-13T07:36:31.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:31 smithi203 bash[58919]: ts=2024-04-13T07:36:31.607Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-13T07:36:34.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: cephadm 2024-04-13T07:36:32.564397+0000 mgr.x (mgr.24547) 39 : cephadm 1 Upgrade: Updating mgr.y 2024-04-13T07:36:34.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: audit 2024-04-13T07:36:32.571840+0000 mon.a (mon.0) 1032 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:34.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: audit 2024-04-13T07:36:32.573848+0000 mon.c (mon.2) 82 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:36:34.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: audit 2024-04-13T07:36:32.574330+0000 mon.a (mon.0) 1033 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:36:34.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: audit 2024-04-13T07:36:32.576257+0000 mon.c (mon.2) 83 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:36:34.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: audit 2024-04-13T07:36:32.577977+0000 mon.c (mon.2) 84 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:34.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: cephadm 2024-04-13T07:36:32.583307+0000 mgr.x (mgr.24547) 40 : cephadm 1 Deploying daemon mgr.y on smithi089 2024-04-13T07:36:34.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: cluster 2024-04-13T07:36:32.617897+0000 mgr.x (mgr.24547) 41 : cluster 0 pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:34.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:33 smithi203 bash[23480]: audit 2024-04-13T07:36:33.105180+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.89:0/1735710060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:34.022 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: cephadm 2024-04-13T07:36:32.564397+0000 mgr.x (mgr.24547) 39 : cephadm 1 Upgrade: Updating mgr.y 2024-04-13T07:36:34.022 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: audit 2024-04-13T07:36:32.571840+0000 mon.a (mon.0) 1032 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: audit 2024-04-13T07:36:32.573848+0000 mon.c (mon.2) 82 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: audit 2024-04-13T07:36:32.574330+0000 mon.a (mon.0) 1033 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: audit 2024-04-13T07:36:32.576257+0000 mon.c (mon.2) 83 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: audit 2024-04-13T07:36:32.577977+0000 mon.c (mon.2) 84 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: cephadm 2024-04-13T07:36:32.583307+0000 mgr.x (mgr.24547) 40 : cephadm 1 Deploying daemon mgr.y on smithi089 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: cluster 2024-04-13T07:36:32.617897+0000 mgr.x (mgr.24547) 41 : cluster 0 pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[31522]: audit 2024-04-13T07:36:33.105180+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.89:0/1735710060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: cephadm 2024-04-13T07:36:32.564397+0000 mgr.x (mgr.24547) 39 : cephadm 1 Upgrade: Updating mgr.y 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: audit 2024-04-13T07:36:32.571840+0000 mon.a (mon.0) 1032 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: audit 2024-04-13T07:36:32.573848+0000 mon.c (mon.2) 82 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:36:34.023 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: audit 2024-04-13T07:36:32.574330+0000 mon.a (mon.0) 1033 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:36:34.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: audit 2024-04-13T07:36:32.576257+0000 mon.c (mon.2) 83 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:36:34.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: audit 2024-04-13T07:36:32.577977+0000 mon.c (mon.2) 84 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:34.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: cephadm 2024-04-13T07:36:32.583307+0000 mgr.x (mgr.24547) 40 : cephadm 1 Deploying daemon mgr.y on smithi089 2024-04-13T07:36:34.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: cluster 2024-04-13T07:36:32.617897+0000 mgr.x (mgr.24547) 41 : cluster 0 pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:34.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:33 smithi089 bash[21353]: audit 2024-04-13T07:36:33.105180+0000 mon.c (mon.2) 85 : audit 0 from='client.? 172.21.15.89:0/1735710060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:34.645 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:34.646 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:34.646 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:34.646 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:34.647 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:34.647 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:34.647 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:34.648 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:34.648 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:35.035 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:34 smithi089 systemd[1]: Stopping Ceph mgr.y for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:36:35.637 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:35 smithi089 bash[74581]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mgr-y 2024-04-13T07:36:35.638 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y.service: Main process exited, code=exited, status=143/n/a 2024-04-13T07:36:35.638 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:35 smithi089 bash[74665]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mgr-y 2024-04-13T07:36:35.638 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y.service: Failed with result 'exit-code'. 2024-04-13T07:36:35.638 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: Stopped Ceph mgr.y for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:36:36.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:35 smithi203 bash[23480]: cluster 2024-04-13T07:36:34.618760+0000 mgr.x (mgr.24547) 42 : cluster 0 pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:36.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:35 smithi203 bash[23480]: audit 2024-04-13T07:36:35.562864+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.89:0/3498612192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:36.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: Started Ceph mgr.y for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:36:36.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.049 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.049 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.049 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.049 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:35 smithi089 bash[31522]: cluster 2024-04-13T07:36:34.618760+0000 mgr.x (mgr.24547) 42 : cluster 0 pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:36.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:35 smithi089 bash[31522]: audit 2024-04-13T07:36:35.562864+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.89:0/3498612192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:36.050 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:35 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:36:36.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:35 smithi089 bash[21353]: cluster 2024-04-13T07:36:34.618760+0000 mgr.x (mgr.24547) 42 : cluster 0 pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:36.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:35 smithi089 bash[21353]: audit 2024-04-13T07:36:35.562864+0000 mon.c (mon.2) 86 : audit 0 from='client.? 172.21.15.89:0/3498612192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:37.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:36 smithi089 bash[31522]: audit 2024-04-13T07:36:35.812584+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:37.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:36 smithi089 bash[31522]: audit 2024-04-13T07:36:35.821004+0000 mon.a (mon.0) 1035 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:37.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:36 smithi089 bash[31522]: audit 2024-04-13T07:36:35.854947+0000 mon.c (mon.2) 87 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:37.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:36 smithi089 bash[21353]: audit 2024-04-13T07:36:35.812584+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:37.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:36 smithi089 bash[21353]: audit 2024-04-13T07:36:35.821004+0000 mon.a (mon.0) 1035 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:37.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:36 smithi089 bash[21353]: audit 2024-04-13T07:36:35.854947+0000 mon.c (mon.2) 87 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:37.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:36 smithi089 bash[74725]: debug 2024-04-13T07:36:36.822+0000 7f31c5ecd700 1 -- 172.21.15.89:0/2579857194 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55d1414585a0 con 0x55d1421d8c00 2024-04-13T07:36:37.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:36 smithi089 bash[74725]: debug 2024-04-13T07:36:36.958+0000 7f31d00dd200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:36:37.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:36 smithi203 bash[23480]: audit 2024-04-13T07:36:35.812584+0000 mon.a (mon.0) 1034 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:37.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:36 smithi203 bash[23480]: audit 2024-04-13T07:36:35.821004+0000 mon.a (mon.0) 1035 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:37.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:36 smithi203 bash[23480]: audit 2024-04-13T07:36:35.854947+0000 mon.c (mon.2) 87 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:38.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:37 smithi203 bash[23480]: cluster 2024-04-13T07:36:36.619720+0000 mgr.x (mgr.24547) 43 : cluster 0 pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:38.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:37 smithi089 bash[31522]: cluster 2024-04-13T07:36:36.619720+0000 mgr.x (mgr.24547) 43 : cluster 0 pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:38.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:37 smithi089 bash[21353]: cluster 2024-04-13T07:36:36.619720+0000 mgr.x (mgr.24547) 43 : cluster 0 pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:38.824 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:38 smithi089 bash[74725]: debug 2024-04-13T07:36:38.562+0000 7f31d00dd200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:36:38.824 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:38 smithi089 bash[74725]: debug 2024-04-13T07:36:38.658+0000 7f31d00dd200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:36:39.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:38 smithi203 bash[23480]: audit 2024-04-13T07:36:38.038002+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.89:0/1205292798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:39.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:38 smithi089 bash[21353]: audit 2024-04-13T07:36:38.038002+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.89:0/1205292798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:39.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:38 smithi089 bash[74725]: debug 2024-04-13T07:36:38.918+0000 7f31d00dd200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:36:39.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:38 smithi089 bash[31522]: audit 2024-04-13T07:36:38.038002+0000 mon.c (mon.2) 88 : audit 0 from='client.? 172.21.15.89:0/1205292798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:40.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:39 smithi089 bash[21353]: cluster 2024-04-13T07:36:38.620355+0000 mgr.x (mgr.24547) 44 : cluster 0 pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:40.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:39 smithi089 bash[74725]: debug 2024-04-13T07:36:39.730+0000 7f31d00dd200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:36:40.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:39 smithi089 bash[74725]: debug 2024-04-13T07:36:39.842+0000 7f31d00dd200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:36:40.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:39 smithi089 bash[74725]: debug 2024-04-13T07:36:39.950+0000 7f31d00dd200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:36:40.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:39 smithi089 bash[31522]: cluster 2024-04-13T07:36:38.620355+0000 mgr.x (mgr.24547) 44 : cluster 0 pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:40.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:39 smithi203 bash[23480]: cluster 2024-04-13T07:36:38.620355+0000 mgr.x (mgr.24547) 44 : cluster 0 pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:40.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:40 smithi089 bash[74725]: debug 2024-04-13T07:36:40.322+0000 7f31d00dd200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:36:40.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:40 smithi089 bash[74725]: debug 2024-04-13T07:36:40.438+0000 7f31d00dd200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:36:40.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:40 smithi089 bash[74725]: debug 2024-04-13T07:36:40.546+0000 7f31d00dd200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:36:41.172 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:40 smithi089 bash[74725]: debug 2024-04-13T07:36:40.918+0000 7f31d00dd200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:36:41.173 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:40 smithi089 bash[21353]: audit 2024-04-13T07:36:40.468846+0000 mon.a (mon.0) 1036 : audit 0 from='client.? 172.21.15.89:0/311026729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:41.173 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:40 smithi089 bash[31522]: audit 2024-04-13T07:36:40.468846+0000 mon.a (mon.0) 1036 : audit 0 from='client.? 172.21.15.89:0/311026729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:41.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:40 smithi203 bash[23480]: audit 2024-04-13T07:36:40.468846+0000 mon.a (mon.0) 1036 : audit 0 from='client.? 172.21.15.89:0/311026729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:41.455 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[74725]: debug 2024-04-13T07:36:41.178+0000 7f31d00dd200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:36:41.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[74725]: debug 2024-04-13T07:36:41.458+0000 7f31d00dd200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:36:41.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[74725]: debug 2024-04-13T07:36:41.586+0000 7f31d00dd200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:36:42.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:41 smithi203 bash[23480]: cluster 2024-04-13T07:36:40.621516+0000 mgr.x (mgr.24547) 45 : cluster 0 pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:42.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:41 smithi203 bash[23480]: audit 2024-04-13T07:36:41.643162+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:42.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:41 smithi203 bash[23480]: audit 2024-04-13T07:36:41.649961+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:42.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[74725]: debug 2024-04-13T07:36:41.886+0000 7f31d00dd200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:36:42.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[31522]: cluster 2024-04-13T07:36:40.621516+0000 mgr.x (mgr.24547) 45 : cluster 0 pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:42.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[31522]: audit 2024-04-13T07:36:41.643162+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:42.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[31522]: audit 2024-04-13T07:36:41.649961+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[21353]: cluster 2024-04-13T07:36:40.621516+0000 mgr.x (mgr.24547) 45 : cluster 0 pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[21353]: audit 2024-04-13T07:36:41.643162+0000 mon.a (mon.0) 1037 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:41 smithi089 bash[21353]: audit 2024-04-13T07:36:41.649961+0000 mon.a (mon.0) 1038 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:42.579 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[74725]: debug 2024-04-13T07:36:42.322+0000 7f31d00dd200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:36:42.941 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[74725]: debug 2024-04-13T07:36:42.582+0000 7f31d00dd200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:36:42.941 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[74725]: debug 2024-04-13T07:36:42.718+0000 7f31d00dd200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:36:43.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:42 smithi203 bash[23480]: audit 2024-04-13T07:36:41.949291+0000 mon.a (mon.0) 1039 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:42 smithi203 bash[23480]: audit 2024-04-13T07:36:41.962231+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:42 smithi203 bash[23480]: audit 2024-04-13T07:36:42.754666+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:42 smithi203 bash[23480]: audit 2024-04-13T07:36:42.766875+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:42 smithi203 bash[23480]: audit 2024-04-13T07:36:42.908749+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.89:0/3011692178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:43.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[74725]: debug 2024-04-13T07:36:42.986+0000 7f31d00dd200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[31522]: audit 2024-04-13T07:36:41.949291+0000 mon.a (mon.0) 1039 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[31522]: audit 2024-04-13T07:36:41.962231+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[31522]: audit 2024-04-13T07:36:42.754666+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[31522]: audit 2024-04-13T07:36:42.766875+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[31522]: audit 2024-04-13T07:36:42.908749+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.89:0/3011692178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[21353]: audit 2024-04-13T07:36:41.949291+0000 mon.a (mon.0) 1039 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[21353]: audit 2024-04-13T07:36:41.962231+0000 mon.a (mon.0) 1040 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[21353]: audit 2024-04-13T07:36:42.754666+0000 mon.a (mon.0) 1041 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[21353]: audit 2024-04-13T07:36:42.766875+0000 mon.a (mon.0) 1042 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:43.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:42 smithi089 bash[21353]: audit 2024-04-13T07:36:42.908749+0000 mon.c (mon.2) 89 : audit 0 from='client.? 172.21.15.89:0/3011692178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:43.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:43 smithi203 bash[58919]: ts=2024-04-13T07:36:43.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi203\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi203\", job=\"node\", machine=\"x86_64\", nodename=\"smithi203\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi203\", job=\"node\", machine=\"x86_64\", nodename=\"smithi203\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:36:44.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:43 smithi203 bash[23480]: cluster 2024-04-13T07:36:42.622261+0000 mgr.x (mgr.24547) 46 : cluster 0 pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:44.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:43 smithi203 bash[23480]: audit 2024-04-13T07:36:43.713059+0000 mon.c (mon.2) 90 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:44.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:43 smithi089 bash[31522]: cluster 2024-04-13T07:36:42.622261+0000 mgr.x (mgr.24547) 46 : cluster 0 pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:44.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:43 smithi089 bash[31522]: audit 2024-04-13T07:36:43.713059+0000 mon.c (mon.2) 90 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:43 smithi089 bash[21353]: cluster 2024-04-13T07:36:42.622261+0000 mgr.x (mgr.24547) 46 : cluster 0 pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:44.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:43 smithi089 bash[21353]: audit 2024-04-13T07:36:43.713059+0000 mon.c (mon.2) 90 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:44.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:44 smithi089 bash[74725]: debug 2024-04-13T07:36:44.034+0000 7f31d00dd200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:36:44.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:44 smithi089 bash[74725]: debug 2024-04-13T07:36:44.150+0000 7f31d00dd200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:36:44.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:44 smithi203 bash[52286]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:36:44] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-13T07:36:44.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:44 smithi089 bash[74725]: debug 2024-04-13T07:36:44.462+0000 7f31d00dd200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:36:44.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:44 smithi089 bash[74725]: debug 2024-04-13T07:36:44.578+0000 7f31d00dd200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:36:45.307 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[74725]: debug 2024-04-13T07:36:45.054+0000 7f31d00dd200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:36:45.307 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[74725]: debug 2024-04-13T07:36:45.182+0000 7f31d00dd200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:36:45.307 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[74725]: [13/Apr/2024:07:36:45] ENGINE Bus STARTING 2024-04-13T07:36:45.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[74725]: CherryPy Checker: 2024-04-13T07:36:45.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[74725]: The Application mounted at '' has an empty config. 2024-04-13T07:36:45.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[74725]: [13/Apr/2024:07:36:45] ENGINE Serving on http://:::9283 2024-04-13T07:36:45.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[74725]: [13/Apr/2024:07:36:45] ENGINE Bus STARTED 2024-04-13T07:36:46.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:45 smithi203 bash[23480]: cluster 2024-04-13T07:36:44.623125+0000 mgr.x (mgr.24547) 47 : cluster 0 pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:46.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:45 smithi203 bash[23480]: cluster 2024-04-13T07:36:45.205185+0000 mon.a (mon.0) 1043 : cluster 0 Standby manager daemon y restarted 2024-04-13T07:36:46.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:45 smithi203 bash[23480]: cluster 2024-04-13T07:36:45.205357+0000 mon.a (mon.0) 1044 : cluster 0 Standby manager daemon y started 2024-04-13T07:36:46.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:45 smithi203 bash[23480]: audit 2024-04-13T07:36:45.207178+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-13T07:36:46.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:45 smithi203 bash[23480]: audit 2024-04-13T07:36:45.208150+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:36:46.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:45 smithi203 bash[23480]: audit 2024-04-13T07:36:45.214086+0000 mon.a (mon.0) 1047 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-13T07:36:46.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:45 smithi203 bash[23480]: audit 2024-04-13T07:36:45.214608+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:36:46.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:45 smithi203 bash[23480]: audit 2024-04-13T07:36:45.338839+0000 mon.a (mon.0) 1049 : audit 0 from='client.? 172.21.15.89:0/4026044517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:46.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[31522]: cluster 2024-04-13T07:36:44.623125+0000 mgr.x (mgr.24547) 47 : cluster 0 pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:46.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[31522]: cluster 2024-04-13T07:36:45.205185+0000 mon.a (mon.0) 1043 : cluster 0 Standby manager daemon y restarted 2024-04-13T07:36:46.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[31522]: cluster 2024-04-13T07:36:45.205357+0000 mon.a (mon.0) 1044 : cluster 0 Standby manager daemon y started 2024-04-13T07:36:46.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[31522]: audit 2024-04-13T07:36:45.207178+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-13T07:36:46.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[31522]: audit 2024-04-13T07:36:45.208150+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:36:46.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[31522]: audit 2024-04-13T07:36:45.214086+0000 mon.a (mon.0) 1047 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-13T07:36:46.366 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[31522]: audit 2024-04-13T07:36:45.214608+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[31522]: audit 2024-04-13T07:36:45.338839+0000 mon.a (mon.0) 1049 : audit 0 from='client.? 172.21.15.89:0/4026044517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[21353]: cluster 2024-04-13T07:36:44.623125+0000 mgr.x (mgr.24547) 47 : cluster 0 pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[21353]: cluster 2024-04-13T07:36:45.205185+0000 mon.a (mon.0) 1043 : cluster 0 Standby manager daemon y restarted 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[21353]: cluster 2024-04-13T07:36:45.205357+0000 mon.a (mon.0) 1044 : cluster 0 Standby manager daemon y started 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[21353]: audit 2024-04-13T07:36:45.207178+0000 mon.a (mon.0) 1045 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[21353]: audit 2024-04-13T07:36:45.208150+0000 mon.a (mon.0) 1046 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[21353]: audit 2024-04-13T07:36:45.214086+0000 mon.a (mon.0) 1047 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[21353]: audit 2024-04-13T07:36:45.214608+0000 mon.a (mon.0) 1048 : audit 0 from='mgr.? 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:36:46.367 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:45 smithi089 bash[21353]: audit 2024-04-13T07:36:45.338839+0000 mon.a (mon.0) 1049 : audit 0 from='client.? 172.21.15.89:0/4026044517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:47.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:47 smithi203 bash[23480]: cluster 2024-04-13T07:36:45.982602+0000 mon.a (mon.0) 1050 : cluster 0 mgrmap e26: x(active, since 47s), standbys: y 2024-04-13T07:36:47.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:47 smithi089 bash[31522]: cluster 2024-04-13T07:36:45.982602+0000 mon.a (mon.0) 1050 : cluster 0 mgrmap e26: x(active, since 47s), standbys: y 2024-04-13T07:36:47.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:46 smithi089 bash[21353]: cluster 2024-04-13T07:36:45.982602+0000 mon.a (mon.0) 1050 : cluster 0 mgrmap e26: x(active, since 47s), standbys: y 2024-04-13T07:36:48.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:48 smithi089 bash[31522]: cluster 2024-04-13T07:36:46.624180+0000 mgr.x (mgr.24547) 48 : cluster 0 pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:48.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:48 smithi089 bash[31522]: audit 2024-04-13T07:36:47.789272+0000 mon.a (mon.0) 1051 : audit 0 from='client.? 172.21.15.89:0/4028911128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:48.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:48 smithi089 bash[21353]: cluster 2024-04-13T07:36:46.624180+0000 mgr.x (mgr.24547) 48 : cluster 0 pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:48.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:48 smithi089 bash[21353]: audit 2024-04-13T07:36:47.789272+0000 mon.a (mon.0) 1051 : audit 0 from='client.? 172.21.15.89:0/4028911128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:48.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:48 smithi203 bash[23480]: cluster 2024-04-13T07:36:46.624180+0000 mgr.x (mgr.24547) 48 : cluster 0 pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:48.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:48 smithi203 bash[23480]: audit 2024-04-13T07:36:47.789272+0000 mon.a (mon.0) 1051 : audit 0 from='client.? 172.21.15.89:0/4028911128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:50.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:50 smithi203 bash[23480]: cluster 2024-04-13T07:36:48.624990+0000 mgr.x (mgr.24547) 49 : cluster 0 pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:50.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:50 smithi089 bash[21353]: cluster 2024-04-13T07:36:48.624990+0000 mgr.x (mgr.24547) 49 : cluster 0 pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:50.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:50 smithi089 bash[31522]: cluster 2024-04-13T07:36:48.624990+0000 mgr.x (mgr.24547) 49 : cluster 0 pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:51.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:51 smithi203 bash[23480]: audit 2024-04-13T07:36:50.232313+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.89:0/2659103723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:51.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:51 smithi089 bash[31522]: audit 2024-04-13T07:36:50.232313+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.89:0/2659103723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:51.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:51 smithi089 bash[21353]: audit 2024-04-13T07:36:50.232313+0000 mon.c (mon.2) 91 : audit 0 from='client.? 172.21.15.89:0/2659103723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:52.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:52 smithi203 bash[23480]: cluster 2024-04-13T07:36:50.626116+0000 mgr.x (mgr.24547) 50 : cluster 0 pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:52.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:52 smithi203 bash[52286]: ignoring --setuser ceph since I am not root 2024-04-13T07:36:52.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:52 smithi203 bash[52286]: ignoring --setgroup ceph since I am not root 2024-04-13T07:36:52.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:52 smithi203 bash[52286]: debug 2024-04-13T07:36:52.257+0000 7fca0287c700 1 -- 172.21.15.203:0/2011877835 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5630554da5a0 con 0x563056258c00 2024-04-13T07:36:52.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[31522]: cluster 2024-04-13T07:36:50.626116+0000 mgr.x (mgr.24547) 50 : cluster 0 pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:52.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[21353]: cluster 2024-04-13T07:36:50.626116+0000 mgr.x (mgr.24547) 50 : cluster 0 pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:52.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: [13/Apr/2024:07:36:52] ENGINE Bus STOPPING 2024-04-13T07:36:52.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: [13/Apr/2024:07:36:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-13T07:36:52.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: [13/Apr/2024:07:36:52] ENGINE Bus STOPPED 2024-04-13T07:36:52.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:52 smithi203 bash[52286]: debug 2024-04-13T07:36:52.405+0000 7fca0c66e200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:36:52.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: [13/Apr/2024:07:36:52] ENGINE Bus STARTING 2024-04-13T07:36:52.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.390+0000 7f3131cd1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.390+0000 7f3131cd1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.390+0000 7f3131cd1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.390+0000 7f3131cd1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.798 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.390+0000 7f3131cd1700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.394+0000 7f312dcc9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.394+0000 7f312dcc9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.394+0000 7f312dcc9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.394+0000 7f312dcc9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: debug 2024-04-13T07:36:52.394+0000 7f312dcc9700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:36:52.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: [13/Apr/2024:07:36:52] ENGINE Serving on http://:::9283 2024-04-13T07:36:52.799 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:36:52 smithi089 bash[74725]: [13/Apr/2024:07:36:52] ENGINE Bus STARTED 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.031467+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.048062+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.052491+0000 mon.c (mon.2) 92 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.054747+0000 mon.c (mon.2) 93 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.066278+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.164164+0000 mon.c (mon.2) 94 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.176708+0000 mon.c (mon.2) 95 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.177412+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: cluster 2024-04-13T07:36:52.178298+0000 mon.a (mon.0) 1056 : cluster 1 Activating manager daemon y 2024-04-13T07:36:53.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: cluster 2024-04-13T07:36:52.195456+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.196062+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: cluster 2024-04-13T07:36:52.196248+0000 mon.a (mon.0) 1059 : cluster 0 mgrmap e27: y(active, starting, since 0.0179418s) 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.203666+0000 mon.c (mon.2) 96 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.204145+0000 mon.c (mon.2) 97 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.204740+0000 mon.c (mon.2) 98 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.205244+0000 mon.c (mon.2) 99 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.205846+0000 mon.c (mon.2) 100 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.206477+0000 mon.c (mon.2) 101 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.207131+0000 mon.c (mon.2) 102 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.207878+0000 mon.c (mon.2) 103 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:36:53.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.031467+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.048062+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.052491+0000 mon.c (mon.2) 92 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.054747+0000 mon.c (mon.2) 93 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.066278+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.164164+0000 mon.c (mon.2) 94 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.176708+0000 mon.c (mon.2) 95 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.177412+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: cluster 2024-04-13T07:36:52.178298+0000 mon.a (mon.0) 1056 : cluster 1 Activating manager daemon y 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: cluster 2024-04-13T07:36:52.195456+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.196062+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: cluster 2024-04-13T07:36:52.196248+0000 mon.a (mon.0) 1059 : cluster 0 mgrmap e27: y(active, starting, since 0.0179418s) 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.203666+0000 mon.c (mon.2) 96 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.204145+0000 mon.c (mon.2) 97 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.204740+0000 mon.c (mon.2) 98 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:36:53.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.205244+0000 mon.c (mon.2) 99 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.205846+0000 mon.c (mon.2) 100 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.206477+0000 mon.c (mon.2) 101 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.207131+0000 mon.c (mon.2) 102 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.207878+0000 mon.c (mon.2) 103 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.208523+0000 mon.c (mon.2) 104 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.209579+0000 mon.c (mon.2) 105 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.210370+0000 mon.c (mon.2) 106 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.211175+0000 mon.c (mon.2) 107 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.211772+0000 mon.c (mon.2) 108 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:36:53.302 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.212462+0000 mon.c (mon.2) 109 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.213176+0000 mon.c (mon.2) 110 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.213837+0000 mon.c (mon.2) 111 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.215147+0000 mon.c (mon.2) 112 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: cluster 2024-04-13T07:36:52.259073+0000 mon.a (mon.0) 1060 : cluster 1 Manager daemon y is now available 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.298097+0000 mon.c (mon.2) 113 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.324082+0000 mon.c (mon.2) 114 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.339266+0000 mon.c (mon.2) 115 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.339682+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.380242+0000 mon.c (mon.2) 116 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.380696+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[21353]: audit 2024-04-13T07:36:52.747000+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.89:0/92283772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.208523+0000 mon.c (mon.2) 104 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:36:53.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.209579+0000 mon.c (mon.2) 105 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.210370+0000 mon.c (mon.2) 106 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.211175+0000 mon.c (mon.2) 107 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.211772+0000 mon.c (mon.2) 108 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.212462+0000 mon.c (mon.2) 109 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.213176+0000 mon.c (mon.2) 110 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.213837+0000 mon.c (mon.2) 111 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.215147+0000 mon.c (mon.2) 112 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: cluster 2024-04-13T07:36:52.259073+0000 mon.a (mon.0) 1060 : cluster 1 Manager daemon y is now available 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.298097+0000 mon.c (mon.2) 113 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.324082+0000 mon.c (mon.2) 114 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.339266+0000 mon.c (mon.2) 115 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.339682+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.380242+0000 mon.c (mon.2) 116 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:53.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.380696+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:53.305 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:53 smithi089 bash[31522]: audit 2024-04-13T07:36:52.747000+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.89:0/92283772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:53.504 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.031467+0000 mon.a (mon.0) 1052 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.504 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.048062+0000 mon.a (mon.0) 1053 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.504 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.052491+0000 mon.c (mon.2) 92 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.054747+0000 mon.c (mon.2) 93 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.066278+0000 mon.a (mon.0) 1054 : audit 1 from='mgr.24547 ' entity='mgr.x' 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.164164+0000 mon.c (mon.2) 94 : audit 0 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.176708+0000 mon.c (mon.2) 95 : audit 1 from='mgr.24547 172.21.15.203:0/126148574' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.177412+0000 mon.a (mon.0) 1055 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: cluster 2024-04-13T07:36:52.178298+0000 mon.a (mon.0) 1056 : cluster 1 Activating manager daemon y 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: cluster 2024-04-13T07:36:52.195456+0000 mon.a (mon.0) 1057 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.196062+0000 mon.a (mon.0) 1058 : audit 1 from='mgr.24547 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: cluster 2024-04-13T07:36:52.196248+0000 mon.a (mon.0) 1059 : cluster 0 mgrmap e27: y(active, starting, since 0.0179418s) 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.203666+0000 mon.c (mon.2) 96 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.204145+0000 mon.c (mon.2) 97 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:36:53.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.204740+0000 mon.c (mon.2) 98 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.205244+0000 mon.c (mon.2) 99 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.205846+0000 mon.c (mon.2) 100 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.206477+0000 mon.c (mon.2) 101 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.207131+0000 mon.c (mon.2) 102 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.207878+0000 mon.c (mon.2) 103 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.208523+0000 mon.c (mon.2) 104 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.209579+0000 mon.c (mon.2) 105 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.210370+0000 mon.c (mon.2) 106 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.211175+0000 mon.c (mon.2) 107 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.211772+0000 mon.c (mon.2) 108 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.212462+0000 mon.c (mon.2) 109 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:36:53.506 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.213176+0000 mon.c (mon.2) 110 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.213837+0000 mon.c (mon.2) 111 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.215147+0000 mon.c (mon.2) 112 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: cluster 2024-04-13T07:36:52.259073+0000 mon.a (mon.0) 1060 : cluster 1 Manager daemon y is now available 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.298097+0000 mon.c (mon.2) 113 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.324082+0000 mon.c (mon.2) 114 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.339266+0000 mon.c (mon.2) 115 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.339682+0000 mon.a (mon.0) 1061 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.380242+0000 mon.c (mon.2) 116 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.380696+0000 mon.a (mon.0) 1062 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:36:53.507 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[23480]: audit 2024-04-13T07:36:52.747000+0000 mon.c (mon.2) 117 : audit 0 from='client.? 172.21.15.89:0/92283772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:53.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:36:53 smithi203 bash[58919]: ts=2024-04-13T07:36:53.513Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:36:54.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[52286]: debug 2024-04-13T07:36:54.005+0000 7fca0c66e200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:36:54.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[52286]: debug 2024-04-13T07:36:54.105+0000 7fca0c66e200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:36:54.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[23480]: cephadm 2024-04-13T07:36:53.138093+0000 mgr.y (mgr.15744) 1 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Bus STARTING 2024-04-13T07:36:54.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[23480]: cluster 2024-04-13T07:36:53.190696+0000 mon.a (mon.0) 1063 : cluster 0 mgrmap e28: y(active, since 1.0124s) 2024-04-13T07:36:54.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[23480]: cluster 2024-04-13T07:36:53.211178+0000 mgr.y (mgr.15744) 2 : cluster 0 pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:54.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[23480]: cephadm 2024-04-13T07:36:53.253078+0000 mgr.y (mgr.15744) 3 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:36:54.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[23480]: cephadm 2024-04-13T07:36:53.354619+0000 mgr.y (mgr.15744) 4 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:36:54.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[23480]: cephadm 2024-04-13T07:36:53.354775+0000 mgr.y (mgr.15744) 5 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Bus STARTED 2024-04-13T07:36:54.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[21353]: cephadm 2024-04-13T07:36:53.138093+0000 mgr.y (mgr.15744) 1 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Bus STARTING 2024-04-13T07:36:54.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[21353]: cluster 2024-04-13T07:36:53.190696+0000 mon.a (mon.0) 1063 : cluster 0 mgrmap e28: y(active, since 1.0124s) 2024-04-13T07:36:54.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[21353]: cluster 2024-04-13T07:36:53.211178+0000 mgr.y (mgr.15744) 2 : cluster 0 pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:54.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[21353]: cephadm 2024-04-13T07:36:53.253078+0000 mgr.y (mgr.15744) 3 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:36:54.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[21353]: cephadm 2024-04-13T07:36:53.354619+0000 mgr.y (mgr.15744) 4 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:36:54.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[21353]: cephadm 2024-04-13T07:36:53.354775+0000 mgr.y (mgr.15744) 5 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Bus STARTED 2024-04-13T07:36:54.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[31522]: cephadm 2024-04-13T07:36:53.138093+0000 mgr.y (mgr.15744) 1 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Bus STARTING 2024-04-13T07:36:54.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[31522]: cluster 2024-04-13T07:36:53.190696+0000 mon.a (mon.0) 1063 : cluster 0 mgrmap e28: y(active, since 1.0124s) 2024-04-13T07:36:54.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[31522]: cluster 2024-04-13T07:36:53.211178+0000 mgr.y (mgr.15744) 2 : cluster 0 pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:54.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[31522]: cephadm 2024-04-13T07:36:53.253078+0000 mgr.y (mgr.15744) 3 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:36:54.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[31522]: cephadm 2024-04-13T07:36:53.354619+0000 mgr.y (mgr.15744) 4 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:36:54.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:54 smithi089 bash[31522]: cephadm 2024-04-13T07:36:53.354775+0000 mgr.y (mgr.15744) 5 : cephadm 1 [13/Apr/2024:07:36:53] ENGINE Bus STARTED 2024-04-13T07:36:54.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:54 smithi203 bash[52286]: debug 2024-04-13T07:36:54.365+0000 7fca0c66e200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:36:55.512 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:55 smithi203 bash[23480]: cluster 2024-04-13T07:36:54.202212+0000 mon.a (mon.0) 1064 : cluster 0 mgrmap e29: y(active, since 2s) 2024-04-13T07:36:55.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:55 smithi203 bash[23480]: cluster 2024-04-13T07:36:54.209219+0000 mgr.y (mgr.15744) 6 : cluster 0 pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:55.513 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:55 smithi203 bash[52286]: debug 2024-04-13T07:36:55.173+0000 7fca0c66e200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:36:55.513 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:55 smithi203 bash[52286]: debug 2024-04-13T07:36:55.285+0000 7fca0c66e200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:36:55.513 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:55 smithi203 bash[52286]: debug 2024-04-13T07:36:55.389+0000 7fca0c66e200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:36:55.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:55 smithi089 bash[21353]: cluster 2024-04-13T07:36:54.202212+0000 mon.a (mon.0) 1064 : cluster 0 mgrmap e29: y(active, since 2s) 2024-04-13T07:36:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:55 smithi089 bash[21353]: cluster 2024-04-13T07:36:54.209219+0000 mgr.y (mgr.15744) 6 : cluster 0 pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:55.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:55 smithi089 bash[31522]: cluster 2024-04-13T07:36:54.202212+0000 mon.a (mon.0) 1064 : cluster 0 mgrmap e29: y(active, since 2s) 2024-04-13T07:36:55.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:55 smithi089 bash[31522]: cluster 2024-04-13T07:36:54.209219+0000 mgr.y (mgr.15744) 6 : cluster 0 pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:56.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:55 smithi203 bash[52286]: debug 2024-04-13T07:36:55.753+0000 7fca0c66e200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:36:56.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:55 smithi203 bash[52286]: debug 2024-04-13T07:36:55.869+0000 7fca0c66e200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:36:56.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:55 smithi203 bash[52286]: debug 2024-04-13T07:36:55.973+0000 7fca0c66e200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:36:56.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:56 smithi203 bash[52286]: debug 2024-04-13T07:36:56.341+0000 7fca0c66e200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:36:56.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:56 smithi203 bash[23480]: audit 2024-04-13T07:36:55.194678+0000 mon.a (mon.0) 1065 : audit 0 from='client.? 172.21.15.89:0/2171022405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:56.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:56 smithi089 bash[21353]: audit 2024-04-13T07:36:55.194678+0000 mon.a (mon.0) 1065 : audit 0 from='client.? 172.21.15.89:0/2171022405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:56.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:56 smithi089 bash[31522]: audit 2024-04-13T07:36:55.194678+0000 mon.a (mon.0) 1065 : audit 0 from='client.? 172.21.15.89:0/2171022405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:56.844 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:56 smithi203 bash[52286]: debug 2024-04-13T07:36:56.585+0000 7fca0c66e200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:36:57.211 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:56 smithi203 bash[52286]: debug 2024-04-13T07:36:56.849+0000 7fca0c66e200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:36:57.211 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:56 smithi203 bash[52286]: debug 2024-04-13T07:36:56.965+0000 7fca0c66e200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:36:57.512 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:57 smithi203 bash[23480]: cluster 2024-04-13T07:36:56.209886+0000 mgr.y (mgr.15744) 7 : cluster 0 pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:57.512 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:57 smithi203 bash[23480]: cluster 2024-04-13T07:36:56.219072+0000 mon.a (mon.0) 1066 : cluster 0 mgrmap e30: y(active, since 4s) 2024-04-13T07:36:57.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:57 smithi203 bash[52286]: debug 2024-04-13T07:36:57.249+0000 7fca0c66e200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:36:57.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:57 smithi089 bash[21353]: cluster 2024-04-13T07:36:56.209886+0000 mgr.y (mgr.15744) 7 : cluster 0 pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:57.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:57 smithi089 bash[21353]: cluster 2024-04-13T07:36:56.219072+0000 mon.a (mon.0) 1066 : cluster 0 mgrmap e30: y(active, since 4s) 2024-04-13T07:36:57.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:57 smithi089 bash[31522]: cluster 2024-04-13T07:36:56.209886+0000 mgr.y (mgr.15744) 7 : cluster 0 pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:36:57.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:57 smithi089 bash[31522]: cluster 2024-04-13T07:36:56.219072+0000 mon.a (mon.0) 1066 : cluster 0 mgrmap e30: y(active, since 4s) 2024-04-13T07:36:57.917 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:57 smithi203 bash[52286]: debug 2024-04-13T07:36:57.669+0000 7fca0c66e200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:36:58.217 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:57 smithi203 bash[52286]: debug 2024-04-13T07:36:57.921+0000 7fca0c66e200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:36:58.217 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:58 smithi203 bash[52286]: debug 2024-04-13T07:36:58.061+0000 7fca0c66e200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:36:58.217 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:58 smithi203 bash[23480]: audit 2024-04-13T07:36:57.638287+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.89:0/4100374963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:58.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:58 smithi203 bash[52286]: debug 2024-04-13T07:36:58.341+0000 7fca0c66e200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:36:58.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:58 smithi089 bash[31522]: audit 2024-04-13T07:36:57.638287+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.89:0/4100374963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:58 smithi089 bash[21353]: audit 2024-04-13T07:36:57.638287+0000 mon.c (mon.2) 118 : audit 0 from='client.? 172.21.15.89:0/4100374963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:36:59.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[52286]: debug 2024-04-13T07:36:59.433+0000 7fca0c66e200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:36:59.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[52286]: debug 2024-04-13T07:36:59.557+0000 7fca0c66e200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:36:59.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[23480]: cluster 2024-04-13T07:36:58.210418+0000 mgr.y (mgr.15744) 8 : cluster 0 pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-13T07:36:59.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[23480]: audit 2024-04-13T07:36:58.485198+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[23480]: audit 2024-04-13T07:36:58.496020+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[23480]: audit 2024-04-13T07:36:58.793883+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[23480]: audit 2024-04-13T07:36:58.813252+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[31522]: cluster 2024-04-13T07:36:58.210418+0000 mgr.y (mgr.15744) 8 : cluster 0 pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-13T07:36:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[31522]: audit 2024-04-13T07:36:58.485198+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[31522]: audit 2024-04-13T07:36:58.496020+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[31522]: audit 2024-04-13T07:36:58.793883+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[31522]: audit 2024-04-13T07:36:58.813252+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[21353]: cluster 2024-04-13T07:36:58.210418+0000 mgr.y (mgr.15744) 8 : cluster 0 pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 41 KiB/s rd, 0 B/s wr, 23 op/s 2024-04-13T07:36:59.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[21353]: audit 2024-04-13T07:36:58.485198+0000 mon.a (mon.0) 1067 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[21353]: audit 2024-04-13T07:36:58.496020+0000 mon.a (mon.0) 1068 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[21353]: audit 2024-04-13T07:36:58.793883+0000 mon.a (mon.0) 1069 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:36:59.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:36:59 smithi089 bash[21353]: audit 2024-04-13T07:36:58.813252+0000 mon.a (mon.0) 1070 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:00.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[52286]: debug 2024-04-13T07:36:59.873+0000 7fca0c66e200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:37:00.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:36:59 smithi203 bash[52286]: debug 2024-04-13T07:36:59.989+0000 7fca0c66e200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:37:00.697 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[52286]: debug 2024-04-13T07:37:00.449+0000 7fca0c66e200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:37:00.698 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[52286]: debug 2024-04-13T07:37:00.573+0000 7fca0c66e200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:37:00.698 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[52286]: [13/Apr/2024:07:37:00] ENGINE Bus STARTING 2024-04-13T07:37:00.698 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:36:59.610125+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:00.698 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:36:59.618245+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:00.698 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:36:59.888013+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:00.698 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:36:59.903812+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:00.698 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:37:00.076804+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.89:0/1179928740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:00.698 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: cluster 2024-04-13T07:37:00.596299+0000 mon.a (mon.0) 1075 : cluster 0 Standby manager daemon x started 2024-04-13T07:37:00.699 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:37:00.603621+0000 mon.b (mon.1) 34 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:00.699 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:37:00.604906+0000 mon.b (mon.1) 35 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:00.699 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:37:00.607765+0000 mon.b (mon.1) 36 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:00.699 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[23480]: audit 2024-04-13T07:37:00.608202+0000 mon.b (mon.1) 37 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:01.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[52286]: CherryPy Checker: 2024-04-13T07:37:01.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[52286]: The Application mounted at '' has an empty config. 2024-04-13T07:37:01.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[52286]: [13/Apr/2024:07:37:00] ENGINE Serving on http://:::9283 2024-04-13T07:37:01.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:00 smithi203 bash[52286]: [13/Apr/2024:07:37:00] ENGINE Bus STARTED 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:36:59.610125+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:36:59.618245+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:36:59.888013+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:36:59.903812+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:37:00.076804+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.89:0/1179928740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: cluster 2024-04-13T07:37:00.596299+0000 mon.a (mon.0) 1075 : cluster 0 Standby manager daemon x started 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:37:00.603621+0000 mon.b (mon.1) 34 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:37:00.604906+0000 mon.b (mon.1) 35 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:37:00.607765+0000 mon.b (mon.1) 36 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[21353]: audit 2024-04-13T07:37:00.608202+0000 mon.b (mon.1) 37 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:36:59.610125+0000 mon.a (mon.0) 1071 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:36:59.618245+0000 mon.a (mon.0) 1072 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:36:59.888013+0000 mon.a (mon.0) 1073 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:36:59.903812+0000 mon.a (mon.0) 1074 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:37:00.076804+0000 mon.c (mon.2) 119 : audit 0 from='client.? 172.21.15.89:0/1179928740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: cluster 2024-04-13T07:37:00.596299+0000 mon.a (mon.0) 1075 : cluster 0 Standby manager daemon x started 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:37:00.603621+0000 mon.b (mon.1) 34 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:37:00.604906+0000 mon.b (mon.1) 35 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:37:00.607765+0000 mon.b (mon.1) 36 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:00 smithi089 bash[31522]: audit 2024-04-13T07:37:00.608202+0000 mon.b (mon.1) 37 : audit 0 from='mgr.? 172.21.15.203:0/720993038' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:02.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:01 smithi203 bash[23480]: cluster 2024-04-13T07:37:00.211067+0000 mgr.y (mgr.15744) 9 : cluster 0 pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:37:02.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:01 smithi203 bash[23480]: cluster 2024-04-13T07:37:00.926213+0000 mon.a (mon.0) 1076 : cluster 0 mgrmap e31: y(active, since 8s), standbys: x 2024-04-13T07:37:02.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:01 smithi203 bash[23480]: audit 2024-04-13T07:37:00.928739+0000 mon.c (mon.2) 120 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:37:02.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:01 smithi089 bash[31522]: cluster 2024-04-13T07:37:00.211067+0000 mgr.y (mgr.15744) 9 : cluster 0 pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:37:02.091 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:01 smithi089 bash[31522]: cluster 2024-04-13T07:37:00.926213+0000 mon.a (mon.0) 1076 : cluster 0 mgrmap e31: y(active, since 8s), standbys: x 2024-04-13T07:37:02.092 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:01 smithi089 bash[31522]: audit 2024-04-13T07:37:00.928739+0000 mon.c (mon.2) 120 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:37:02.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:01 smithi089 bash[21353]: cluster 2024-04-13T07:37:00.211067+0000 mgr.y (mgr.15744) 9 : cluster 0 pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 29 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:37:02.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:01 smithi089 bash[21353]: cluster 2024-04-13T07:37:00.926213+0000 mon.a (mon.0) 1076 : cluster 0 mgrmap e31: y(active, since 8s), standbys: x 2024-04-13T07:37:02.092 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:01 smithi089 bash[21353]: audit 2024-04-13T07:37:00.928739+0000 mon.c (mon.2) 120 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:37:03.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:02 smithi203 bash[23480]: audit 2024-04-13T07:37:02.525164+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.89:0/564552329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:03.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:02 smithi089 bash[21353]: audit 2024-04-13T07:37:02.525164+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.89:0/564552329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:03.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:02 smithi089 bash[31522]: audit 2024-04-13T07:37:02.525164+0000 mon.c (mon.2) 121 : audit 0 from='client.? 172.21.15.89:0/564552329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:03.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:03 smithi203 bash[23480]: cluster 2024-04-13T07:37:02.211469+0000 mgr.y (mgr.15744) 10 : cluster 0 pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:37:03.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:03 smithi203 bash[58919]: ts=2024-04-13T07:37:03.510Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:37:04.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:03 smithi089 bash[31522]: cluster 2024-04-13T07:37:02.211469+0000 mgr.y (mgr.15744) 10 : cluster 0 pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:37:04.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:03 smithi089 bash[21353]: cluster 2024-04-13T07:37:02.211469+0000 mgr.y (mgr.15744) 10 : cluster 0 pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:37:04.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:04 smithi203 bash[52286]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:37:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-13T07:37:05.769 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:05 smithi089 bash[31522]: cluster 2024-04-13T07:37:04.212613+0000 mgr.y (mgr.15744) 11 : cluster 0 pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:37:05.769 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:05 smithi089 bash[31522]: audit 2024-04-13T07:37:04.965895+0000 mon.a (mon.0) 1077 : audit 0 from='client.? 172.21.15.89:0/3628439419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:05.770 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:05 smithi089 bash[21353]: cluster 2024-04-13T07:37:04.212613+0000 mgr.y (mgr.15744) 11 : cluster 0 pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:37:05.770 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:05 smithi089 bash[21353]: audit 2024-04-13T07:37:04.965895+0000 mon.a (mon.0) 1077 : audit 0 from='client.? 172.21.15.89:0/3628439419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:06.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:05 smithi203 bash[23480]: cluster 2024-04-13T07:37:04.212613+0000 mgr.y (mgr.15744) 11 : cluster 0 pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:37:06.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:05 smithi203 bash[23480]: audit 2024-04-13T07:37:04.965895+0000 mon.a (mon.0) 1077 : audit 0 from='client.? 172.21.15.89:0/3628439419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:08.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:07 smithi203 bash[23480]: cluster 2024-04-13T07:37:06.213241+0000 mgr.y (mgr.15744) 12 : cluster 0 pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:37:08.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:07 smithi203 bash[23480]: audit 2024-04-13T07:37:07.324392+0000 mon.c (mon.2) 122 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:08.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:07 smithi203 bash[23480]: audit 2024-04-13T07:37:07.395704+0000 mon.a (mon.0) 1078 : audit 0 from='client.? 172.21.15.89:0/1530629228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:08.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:07 smithi089 bash[21353]: cluster 2024-04-13T07:37:06.213241+0000 mgr.y (mgr.15744) 12 : cluster 0 pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:37:08.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:07 smithi089 bash[21353]: audit 2024-04-13T07:37:07.324392+0000 mon.c (mon.2) 122 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:08.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:07 smithi089 bash[21353]: audit 2024-04-13T07:37:07.395704+0000 mon.a (mon.0) 1078 : audit 0 from='client.? 172.21.15.89:0/1530629228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:08.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:07 smithi089 bash[31522]: cluster 2024-04-13T07:37:06.213241+0000 mgr.y (mgr.15744) 12 : cluster 0 pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:37:08.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:07 smithi089 bash[31522]: audit 2024-04-13T07:37:07.324392+0000 mon.c (mon.2) 122 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:08.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:07 smithi089 bash[31522]: audit 2024-04-13T07:37:07.395704+0000 mon.a (mon.0) 1078 : audit 0 from='client.? 172.21.15.89:0/1530629228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:10.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: cluster 2024-04-13T07:37:08.214266+0000 mgr.y (mgr.15744) 13 : cluster 0 pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:37:10.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:08.815872+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:08.830804+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:08.835832+0000 mon.c (mon.2) 123 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:10.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:08.837670+0000 mon.c (mon.2) 124 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:37:10.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:09.606693+0000 mon.a (mon.0) 1081 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:09.610101+0000 mon.a (mon.0) 1082 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:09.617446+0000 mon.a (mon.0) 1083 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:09.621937+0000 mon.a (mon.0) 1084 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:09 smithi203 bash[23480]: audit 2024-04-13T07:37:09.628697+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: cluster 2024-04-13T07:37:08.214266+0000 mgr.y (mgr.15744) 13 : cluster 0 pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:08.815872+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:08.830804+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:08.835832+0000 mon.c (mon.2) 123 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:08.837670+0000 mon.c (mon.2) 124 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:09.606693+0000 mon.a (mon.0) 1081 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:09.610101+0000 mon.a (mon.0) 1082 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:09.617446+0000 mon.a (mon.0) 1083 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:09.621937+0000 mon.a (mon.0) 1084 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[21353]: audit 2024-04-13T07:37:09.628697+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: cluster 2024-04-13T07:37:08.214266+0000 mgr.y (mgr.15744) 13 : cluster 0 pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:08.815872+0000 mon.a (mon.0) 1079 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:08.830804+0000 mon.a (mon.0) 1080 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:08.835832+0000 mon.c (mon.2) 123 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:08.837670+0000 mon.c (mon.2) 124 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:09.606693+0000 mon.a (mon.0) 1081 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:09.610101+0000 mon.a (mon.0) 1082 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:09.617446+0000 mon.a (mon.0) 1083 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:09.621937+0000 mon.a (mon.0) 1084 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:10.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:09 smithi089 bash[31522]: audit 2024-04-13T07:37:09.628697+0000 mon.a (mon.0) 1085 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:11.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:08.840348+0000 mgr.y (mgr.15744) 14 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:08.840780+0000 mgr.y (mgr.15744) 15 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:09.106325+0000 mgr.y (mgr.15744) 16 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:09.107574+0000 mgr.y (mgr.15744) 17 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:09.271187+0000 mgr.y (mgr.15744) 18 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:09.273005+0000 mgr.y (mgr.15744) 19 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:09.435481+0000 mgr.y (mgr.15744) 20 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:09.437688+0000 mgr.y (mgr.15744) 21 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: cephadm 2024-04-13T07:37:09.683160+0000 mgr.y (mgr.15744) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:37:11.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:10 smithi203 bash[23480]: audit 2024-04-13T07:37:09.853434+0000 mon.a (mon.0) 1086 : audit 0 from='client.? 172.21.15.89:0/1373187324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:08.840348+0000 mgr.y (mgr.15744) 14 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:08.840780+0000 mgr.y (mgr.15744) 15 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:09.106325+0000 mgr.y (mgr.15744) 16 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:09.107574+0000 mgr.y (mgr.15744) 17 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:09.271187+0000 mgr.y (mgr.15744) 18 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:09.273005+0000 mgr.y (mgr.15744) 19 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:09.435481+0000 mgr.y (mgr.15744) 20 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:09.437688+0000 mgr.y (mgr.15744) 21 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: cephadm 2024-04-13T07:37:09.683160+0000 mgr.y (mgr.15744) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[21353]: audit 2024-04-13T07:37:09.853434+0000 mon.a (mon.0) 1086 : audit 0 from='client.? 172.21.15.89:0/1373187324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:11.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:08.840348+0000 mgr.y (mgr.15744) 14 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:08.840780+0000 mgr.y (mgr.15744) 15 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:09.106325+0000 mgr.y (mgr.15744) 16 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:09.107574+0000 mgr.y (mgr.15744) 17 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:09.271187+0000 mgr.y (mgr.15744) 18 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:09.273005+0000 mgr.y (mgr.15744) 19 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:09.435481+0000 mgr.y (mgr.15744) 20 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:09.437688+0000 mgr.y (mgr.15744) 21 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: cephadm 2024-04-13T07:37:09.683160+0000 mgr.y (mgr.15744) 22 : cephadm 1 Reconfiguring prometheus.a (dependencies changed)... 2024-04-13T07:37:11.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:10 smithi089 bash[31522]: audit 2024-04-13T07:37:09.853434+0000 mon.a (mon.0) 1086 : audit 0 from='client.? 172.21.15.89:0/1373187324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:11.973 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:11 smithi203 bash[23480]: cephadm 2024-04-13T07:37:09.941341+0000 mgr.y (mgr.15744) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:37:11.974 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:11 smithi203 bash[23480]: cluster 2024-04-13T07:37:10.214957+0000 mgr.y (mgr.15744) 24 : cluster 0 pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:11 smithi089 bash[21353]: cephadm 2024-04-13T07:37:09.941341+0000 mgr.y (mgr.15744) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:37:12.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:11 smithi089 bash[21353]: cluster 2024-04-13T07:37:10.214957+0000 mgr.y (mgr.15744) 24 : cluster 0 pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:11 smithi089 bash[31522]: cephadm 2024-04-13T07:37:09.941341+0000 mgr.y (mgr.15744) 23 : cephadm 1 Reconfiguring daemon prometheus.a on smithi203 2024-04-13T07:37:12.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:11 smithi089 bash[31522]: cluster 2024-04-13T07:37:10.214957+0000 mgr.y (mgr.15744) 24 : cluster 0 pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:13.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:12 smithi203 bash[23480]: audit 2024-04-13T07:37:12.302547+0000 mon.a (mon.0) 1087 : audit 0 from='client.? 172.21.15.89:0/3157159496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:13.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:12 smithi089 bash[21353]: audit 2024-04-13T07:37:12.302547+0000 mon.a (mon.0) 1087 : audit 0 from='client.? 172.21.15.89:0/3157159496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:13.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:12 smithi089 bash[31522]: audit 2024-04-13T07:37:12.302547+0000 mon.a (mon.0) 1087 : audit 0 from='client.? 172.21.15.89:0/3157159496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:13.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:13 smithi203 bash[58919]: ts=2024-04-13T07:37:13.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:37:14.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:13 smithi203 bash[23480]: cluster 2024-04-13T07:37:12.215428+0000 mgr.y (mgr.15744) 25 : cluster 0 pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:14.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:13 smithi089 bash[31522]: cluster 2024-04-13T07:37:12.215428+0000 mgr.y (mgr.15744) 25 : cluster 0 pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:14.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:13 smithi089 bash[21353]: cluster 2024-04-13T07:37:12.215428+0000 mgr.y (mgr.15744) 25 : cluster 0 pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:14.747 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[52286]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:37:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 systemd[1]: Stopping Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.505Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.505Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.505Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.505Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.505Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.505Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.505Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.505Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-13T07:37:14.748 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.506Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-13T07:37:14.749 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.506Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-13T07:37:14.749 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.506Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-13T07:37:14.749 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[58919]: ts=2024-04-13T07:37:14.506Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-13T07:37:15.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[23480]: cluster 2024-04-13T07:37:14.216570+0000 mgr.y (mgr.15744) 26 : cluster 0 pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:15.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:14 smithi203 bash[23480]: audit 2024-04-13T07:37:14.734123+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.89:0/3360391772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:15.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:14 smithi089 bash[31522]: cluster 2024-04-13T07:37:14.216570+0000 mgr.y (mgr.15744) 26 : cluster 0 pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:15.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:14 smithi089 bash[31522]: audit 2024-04-13T07:37:14.734123+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.89:0/3360391772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:15.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:14 smithi089 bash[21353]: cluster 2024-04-13T07:37:14.216570+0000 mgr.y (mgr.15744) 26 : cluster 0 pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:15.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:14 smithi089 bash[21353]: audit 2024-04-13T07:37:14.734123+0000 mon.c (mon.2) 125 : audit 0 from='client.? 172.21.15.89:0/3360391772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:15.304 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:15 smithi203 bash[60163]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-prometheus-a 2024-04-13T07:37:15.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:15 smithi203 bash[60224]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-prometheus-a 2024-04-13T07:37:15.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:15 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@prometheus.a.service: Deactivated successfully. 2024-04-13T07:37:15.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:15 smithi203 systemd[1]: Stopped Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:37:15.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:15 smithi203 systemd[1]: Started Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:37:16.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[23480]: audit 2024-04-13T07:37:15.369178+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:16.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[23480]: audit 2024-04-13T07:37:15.384394+0000 mon.a (mon.0) 1089 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:16.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[23480]: audit 2024-04-13T07:37:15.754352+0000 mon.c (mon.2) 126 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:37:16.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[23480]: audit 2024-04-13T07:37:15.755550+0000 mgr.y (mgr.15744) 27 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:37:16.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[23480]: audit 2024-04-13T07:37:15.762543+0000 mon.c (mon.2) 127 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:16.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[21353]: audit 2024-04-13T07:37:15.369178+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[21353]: audit 2024-04-13T07:37:15.384394+0000 mon.a (mon.0) 1089 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[21353]: audit 2024-04-13T07:37:15.754352+0000 mon.c (mon.2) 126 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:37:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[21353]: audit 2024-04-13T07:37:15.755550+0000 mgr.y (mgr.15744) 27 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:37:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[21353]: audit 2024-04-13T07:37:15.762543+0000 mon.c (mon.2) 127 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[31522]: audit 2024-04-13T07:37:15.369178+0000 mon.a (mon.0) 1088 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[31522]: audit 2024-04-13T07:37:15.384394+0000 mon.a (mon.0) 1089 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[31522]: audit 2024-04-13T07:37:15.754352+0000 mon.c (mon.2) 126 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:37:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[31522]: audit 2024-04-13T07:37:15.755550+0000 mgr.y (mgr.15744) 27 : audit 0 from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2024-04-13T07:37:16.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:16 smithi089 bash[31522]: audit 2024-04-13T07:37:15.762543+0000 mon.c (mon.2) 127 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:17.263 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.966Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2024-04-13T07:37:17.263 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.966Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2024-04-13T07:37:17.263 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.966Z caller=main.go:570 level=info host_details="(Linux 5.15.0-102-generic #112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024 x86_64 smithi203 (none))" 2024-04-13T07:37:17.263 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.966Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2024-04-13T07:37:17.263 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.966Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2024-04-13T07:37:17.263 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.968Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.968Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.969Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.969Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.972Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.972Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.856µs 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.972Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.973Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.984Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.991Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.991Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.991Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=25.19µs wal_replay_duration=19.231052ms wbl_replay_duration=147ns total_replay_duration=19.276448ms 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.994Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2024-04-13T07:37:17.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.994Z caller=main.go:1029 level=info msg="TSDB started" 2024-04-13T07:37:17.265 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:16 smithi203 bash[60252]: ts=2024-04-13T07:37:16.994Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2024-04-13T07:37:17.265 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:17 smithi203 bash[60252]: ts=2024-04-13T07:37:17.017Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=23.814854ms db_storage=907ns remote_storage=1.227µs web_handler=526ns query_engine=715ns scrape=252.407µs scrape_sd=107.013µs notify=23.558µs notify_sd=28.67µs rules=23.001261ms tracing=6.045µs 2024-04-13T07:37:17.265 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:17 smithi203 bash[60252]: ts=2024-04-13T07:37:17.017Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2024-04-13T07:37:17.265 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:17 smithi203 bash[60252]: ts=2024-04-13T07:37:17.017Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2024-04-13T07:37:17.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:17 smithi203 bash[23480]: cluster 2024-04-13T07:37:16.217277+0000 mgr.y (mgr.15744) 28 : cluster 0 pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:17.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:17 smithi203 bash[23480]: audit 2024-04-13T07:37:17.173460+0000 mon.a (mon.0) 1090 : audit 0 from='client.? 172.21.15.89:0/3791886402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:17.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:17 smithi089 bash[21353]: cluster 2024-04-13T07:37:16.217277+0000 mgr.y (mgr.15744) 28 : cluster 0 pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:17.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:17 smithi089 bash[21353]: audit 2024-04-13T07:37:17.173460+0000 mon.a (mon.0) 1090 : audit 0 from='client.? 172.21.15.89:0/3791886402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:17.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:17 smithi089 bash[31522]: cluster 2024-04-13T07:37:16.217277+0000 mgr.y (mgr.15744) 28 : cluster 0 pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:17.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:17 smithi089 bash[31522]: audit 2024-04-13T07:37:17.173460+0000 mon.a (mon.0) 1090 : audit 0 from='client.? 172.21.15.89:0/3791886402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:19.686 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:19 smithi203 bash[23480]: cluster 2024-04-13T07:37:18.218271+0000 mgr.y (mgr.15744) 29 : cluster 0 pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:19.687 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:19 smithi203 bash[23480]: cephadm 2024-04-13T07:37:18.386378+0000 mgr.y (mgr.15744) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-04-13T07:37:19.687 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:19 smithi203 bash[23480]: audit 2024-04-13T07:37:18.393183+0000 mon.c (mon.2) 128 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:37:19.687 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:19 smithi203 bash[23480]: audit 2024-04-13T07:37:18.394084+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:37:19.687 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:19 smithi203 bash[23480]: audit 2024-04-13T07:37:18.396621+0000 mon.c (mon.2) 129 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:37:19.687 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:19 smithi203 bash[23480]: audit 2024-04-13T07:37:18.398665+0000 mon.c (mon.2) 130 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:19.687 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:19 smithi203 bash[23480]: cephadm 2024-04-13T07:37:18.400837+0000 mgr.y (mgr.15744) 31 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:37:19.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[21353]: cluster 2024-04-13T07:37:18.218271+0000 mgr.y (mgr.15744) 29 : cluster 0 pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[21353]: cephadm 2024-04-13T07:37:18.386378+0000 mgr.y (mgr.15744) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[21353]: audit 2024-04-13T07:37:18.393183+0000 mon.c (mon.2) 128 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[21353]: audit 2024-04-13T07:37:18.394084+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[21353]: audit 2024-04-13T07:37:18.396621+0000 mon.c (mon.2) 129 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[21353]: audit 2024-04-13T07:37:18.398665+0000 mon.c (mon.2) 130 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[21353]: cephadm 2024-04-13T07:37:18.400837+0000 mgr.y (mgr.15744) 31 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[31522]: cluster 2024-04-13T07:37:18.218271+0000 mgr.y (mgr.15744) 29 : cluster 0 pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[31522]: cephadm 2024-04-13T07:37:18.386378+0000 mgr.y (mgr.15744) 30 : cephadm 1 Upgrade: Updating mgr.x 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[31522]: audit 2024-04-13T07:37:18.393183+0000 mon.c (mon.2) 128 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:37:19.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[31522]: audit 2024-04-13T07:37:18.394084+0000 mon.a (mon.0) 1091 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:37:19.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[31522]: audit 2024-04-13T07:37:18.396621+0000 mon.c (mon.2) 129 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:37:19.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[31522]: audit 2024-04-13T07:37:18.398665+0000 mon.c (mon.2) 130 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:19.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:19 smithi089 bash[31522]: cephadm 2024-04-13T07:37:18.400837+0000 mgr.y (mgr.15744) 31 : cephadm 1 Deploying daemon mgr.x on smithi203 2024-04-13T07:37:20.661 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:20 smithi203 bash[23480]: audit 2024-04-13T07:37:19.615829+0000 mon.a (mon.0) 1092 : audit 0 from='client.? 172.21.15.89:0/4243488022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:20.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:20 smithi089 bash[21353]: audit 2024-04-13T07:37:19.615829+0000 mon.a (mon.0) 1092 : audit 0 from='client.? 172.21.15.89:0/4243488022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:20.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:20 smithi089 bash[31522]: audit 2024-04-13T07:37:19.615829+0000 mon.a (mon.0) 1092 : audit 0 from='client.? 172.21.15.89:0/4243488022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:20.915 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:20.916 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:20.916 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:20.917 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:20.917 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:20.917 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:20.918 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:20.918 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:20.918 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:21.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:20 smithi203 systemd[1]: Stopping Ceph mgr.x for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:37:21.609 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:21 smithi203 bash[23480]: cluster 2024-04-13T07:37:20.218943+0000 mgr.y (mgr.15744) 32 : cluster 0 pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:21.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:21 smithi089 bash[21353]: cluster 2024-04-13T07:37:20.218943+0000 mgr.y (mgr.15744) 32 : cluster 0 pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:21.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:21 smithi089 bash[31522]: cluster 2024-04-13T07:37:20.218943+0000 mgr.y (mgr.15744) 32 : cluster 0 pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:21.967 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:21 smithi203 bash[60535]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mgr-x 2024-04-13T07:37:21.967 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.x.service: Main process exited, code=exited, status=143/n/a 2024-04-13T07:37:21.967 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:21 smithi203 bash[60595]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mgr-x 2024-04-13T07:37:21.967 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.x.service: Failed with result 'exit-code'. 2024-04-13T07:37:21.967 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: Stopped Ceph mgr.x for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:37:22.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.262 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.263 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.263 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.263 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.264 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.264 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.264 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:22 smithi203 systemd[1]: Started Ceph mgr.x for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:37:22.264 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.264 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:37:21 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:22.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:22 smithi203 bash[23480]: audit 2024-04-13T07:37:22.063129+0000 mon.a (mon.0) 1093 : audit 0 from='client.? 172.21.15.89:0/2972171881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:22.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:22 smithi203 bash[23480]: audit 2024-04-13T07:37:22.137917+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:22.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:22 smithi203 bash[23480]: audit 2024-04-13T07:37:22.156707+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:22.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:22 smithi203 bash[23480]: audit 2024-04-13T07:37:22.173483+0000 mon.c (mon.2) 131 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:22.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:22 smithi203 bash[23480]: audit 2024-04-13T07:37:22.325567+0000 mon.c (mon.2) 132 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:22.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[21353]: audit 2024-04-13T07:37:22.063129+0000 mon.a (mon.0) 1093 : audit 0 from='client.? 172.21.15.89:0/2972171881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:22.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[21353]: audit 2024-04-13T07:37:22.137917+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:22.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[21353]: audit 2024-04-13T07:37:22.156707+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[21353]: audit 2024-04-13T07:37:22.173483+0000 mon.c (mon.2) 131 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[21353]: audit 2024-04-13T07:37:22.325567+0000 mon.c (mon.2) 132 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[31522]: audit 2024-04-13T07:37:22.063129+0000 mon.a (mon.0) 1093 : audit 0 from='client.? 172.21.15.89:0/2972171881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[31522]: audit 2024-04-13T07:37:22.137917+0000 mon.a (mon.0) 1094 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[31522]: audit 2024-04-13T07:37:22.156707+0000 mon.a (mon.0) 1095 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[31522]: audit 2024-04-13T07:37:22.173483+0000 mon.c (mon.2) 131 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:22 smithi089 bash[31522]: audit 2024-04-13T07:37:22.325567+0000 mon.c (mon.2) 132 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:23.445 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:23 smithi203 bash[23480]: cluster 2024-04-13T07:37:22.219584+0000 mgr.y (mgr.15744) 33 : cluster 0 pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:23.445 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:23 smithi203 bash[60651]: debug 2024-04-13T07:37:23.300+0000 7fd91e9a5700 1 -- 172.21.15.203:0/2818731977 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x556aa3f665a0 con 0x556aa4ce6c00 2024-04-13T07:37:23.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:23 smithi203 bash[60651]: debug 2024-04-13T07:37:23.448+0000 7fd928bb5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:37:23.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:23 smithi203 bash[60252]: ts=2024-04-13T07:37:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:37:23.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:23 smithi089 bash[21353]: cluster 2024-04-13T07:37:22.219584+0000 mgr.y (mgr.15744) 33 : cluster 0 pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:23.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:23 smithi089 bash[31522]: cluster 2024-04-13T07:37:22.219584+0000 mgr.y (mgr.15744) 33 : cluster 0 pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:25.389 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:25 smithi203 bash[60651]: debug 2024-04-13T07:37:25.044+0000 7fd928bb5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:37:25.389 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:25 smithi203 bash[60651]: debug 2024-04-13T07:37:25.144+0000 7fd928bb5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:37:25.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:25 smithi203 bash[23480]: cluster 2024-04-13T07:37:24.220173+0000 mgr.y (mgr.15744) 34 : cluster 0 pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:25.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:25 smithi203 bash[23480]: audit 2024-04-13T07:37:24.511313+0000 mon.a (mon.0) 1096 : audit 0 from='client.? 172.21.15.89:0/668352202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:25.763 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:25 smithi203 bash[60651]: debug 2024-04-13T07:37:25.392+0000 7fd928bb5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:37:25.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:25 smithi089 bash[31522]: cluster 2024-04-13T07:37:24.220173+0000 mgr.y (mgr.15744) 34 : cluster 0 pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:25.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:25 smithi089 bash[31522]: audit 2024-04-13T07:37:24.511313+0000 mon.a (mon.0) 1096 : audit 0 from='client.? 172.21.15.89:0/668352202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:25.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:25 smithi089 bash[21353]: cluster 2024-04-13T07:37:24.220173+0000 mgr.y (mgr.15744) 34 : cluster 0 pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:25.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:25 smithi089 bash[21353]: audit 2024-04-13T07:37:24.511313+0000 mon.a (mon.0) 1096 : audit 0 from='client.? 172.21.15.89:0/668352202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:26.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:37:25] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-13T07:37:26.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:26 smithi203 bash[60651]: debug 2024-04-13T07:37:26.200+0000 7fd928bb5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:37:26.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:26 smithi203 bash[60651]: debug 2024-04-13T07:37:26.316+0000 7fd928bb5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:37:26.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:26 smithi203 bash[60651]: debug 2024-04-13T07:37:26.420+0000 7fd928bb5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:37:27.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:26 smithi203 bash[60651]: debug 2024-04-13T07:37:26.784+0000 7fd928bb5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:37:27.263 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:26 smithi203 bash[60651]: debug 2024-04-13T07:37:26.900+0000 7fd928bb5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:37:27.263 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:27 smithi203 bash[60651]: debug 2024-04-13T07:37:27.004+0000 7fd928bb5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:37:27.638 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:27 smithi203 bash[60651]: debug 2024-04-13T07:37:27.384+0000 7fd928bb5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:37:27.638 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:27 smithi203 bash[23480]: cluster 2024-04-13T07:37:26.220958+0000 mgr.y (mgr.15744) 35 : cluster 0 pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:27.638 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:27 smithi203 bash[23480]: audit 2024-04-13T07:37:26.948430+0000 mon.a (mon.0) 1097 : audit 0 from='client.? 172.21.15.89:0/4153380546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:27.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:27 smithi089 bash[31522]: cluster 2024-04-13T07:37:26.220958+0000 mgr.y (mgr.15744) 35 : cluster 0 pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:27.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:27 smithi089 bash[31522]: audit 2024-04-13T07:37:26.948430+0000 mon.a (mon.0) 1097 : audit 0 from='client.? 172.21.15.89:0/4153380546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:27 smithi089 bash[21353]: cluster 2024-04-13T07:37:26.220958+0000 mgr.y (mgr.15744) 35 : cluster 0 pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:27 smithi089 bash[21353]: audit 2024-04-13T07:37:26.948430+0000 mon.a (mon.0) 1097 : audit 0 from='client.? 172.21.15.89:0/4153380546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:27.931 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:27 smithi203 bash[60651]: debug 2024-04-13T07:37:27.644+0000 7fd928bb5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:37:28.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:27 smithi203 bash[60651]: debug 2024-04-13T07:37:27.936+0000 7fd928bb5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:37:28.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:28 smithi203 bash[60651]: debug 2024-04-13T07:37:28.060+0000 7fd928bb5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:37:28.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:28 smithi203 bash[60651]: debug 2024-04-13T07:37:28.352+0000 7fd928bb5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:37:29.038 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:28 smithi203 bash[60651]: debug 2024-04-13T07:37:28.792+0000 7fd928bb5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:37:29.431 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:29 smithi203 bash[60651]: debug 2024-04-13T07:37:29.044+0000 7fd928bb5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:37:29.431 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:29 smithi203 bash[60651]: debug 2024-04-13T07:37:29.172+0000 7fd928bb5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:37:29.431 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:29 smithi203 bash[23480]: audit 2024-04-13T07:37:28.170713+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:29.431 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:29 smithi203 bash[23480]: audit 2024-04-13T07:37:28.187019+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:29.431 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:29 smithi203 bash[23480]: cluster 2024-04-13T07:37:28.222366+0000 mgr.y (mgr.15744) 36 : cluster 0 pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:29.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:29 smithi089 bash[31522]: audit 2024-04-13T07:37:28.170713+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:29.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:29 smithi089 bash[31522]: audit 2024-04-13T07:37:28.187019+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:29.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:29 smithi089 bash[31522]: cluster 2024-04-13T07:37:28.222366+0000 mgr.y (mgr.15744) 36 : cluster 0 pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:29.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:29 smithi089 bash[21353]: audit 2024-04-13T07:37:28.170713+0000 mon.a (mon.0) 1098 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:29.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:29 smithi089 bash[21353]: audit 2024-04-13T07:37:28.187019+0000 mon.a (mon.0) 1099 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:29.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:29 smithi089 bash[21353]: cluster 2024-04-13T07:37:28.222366+0000 mgr.y (mgr.15744) 36 : cluster 0 pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:29.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:29 smithi203 bash[60651]: debug 2024-04-13T07:37:29.436+0000 7fd928bb5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:37:30.588 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:30 smithi203 bash[60651]: debug 2024-04-13T07:37:30.476+0000 7fd928bb5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:37:30.589 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:30 smithi203 bash[23480]: audit 2024-04-13T07:37:29.344414+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:30.589 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:30 smithi203 bash[23480]: audit 2024-04-13T07:37:29.359546+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:30.589 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:30 smithi203 bash[23480]: audit 2024-04-13T07:37:29.403413+0000 mon.a (mon.0) 1102 : audit 0 from='client.? 172.21.15.89:0/3171316368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:30 smithi089 bash[21353]: audit 2024-04-13T07:37:29.344414+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:30 smithi089 bash[21353]: audit 2024-04-13T07:37:29.359546+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:30 smithi089 bash[21353]: audit 2024-04-13T07:37:29.403413+0000 mon.a (mon.0) 1102 : audit 0 from='client.? 172.21.15.89:0/3171316368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:30.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:30 smithi089 bash[31522]: audit 2024-04-13T07:37:29.344414+0000 mon.a (mon.0) 1100 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:30.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:30 smithi089 bash[31522]: audit 2024-04-13T07:37:29.359546+0000 mon.a (mon.0) 1101 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:30.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:30 smithi089 bash[31522]: audit 2024-04-13T07:37:29.403413+0000 mon.a (mon.0) 1102 : audit 0 from='client.? 172.21.15.89:0/3171316368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:30.897 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:30 smithi203 bash[60651]: debug 2024-04-13T07:37:30.592+0000 7fd928bb5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:37:31.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:30 smithi203 bash[60651]: debug 2024-04-13T07:37:30.900+0000 7fd928bb5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:37:31.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[60651]: debug 2024-04-13T07:37:31.020+0000 7fd928bb5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:37:31.607 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[23480]: cluster 2024-04-13T07:37:30.223240+0000 mgr.y (mgr.15744) 37 : cluster 0 pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:31.607 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[60651]: debug 2024-04-13T07:37:31.484+0000 7fd928bb5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:37:31.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:31 smithi089 bash[31522]: cluster 2024-04-13T07:37:30.223240+0000 mgr.y (mgr.15744) 37 : cluster 0 pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:31.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:31 smithi089 bash[21353]: cluster 2024-04-13T07:37:30.223240+0000 mgr.y (mgr.15744) 37 : cluster 0 pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:32.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[60651]: debug 2024-04-13T07:37:31.612+0000 7fd928bb5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:37:32.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[60651]: [13/Apr/2024:07:37:31] ENGINE Bus STARTING 2024-04-13T07:37:32.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[60651]: CherryPy Checker: 2024-04-13T07:37:32.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[60651]: The Application mounted at '' has an empty config. 2024-04-13T07:37:32.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[60651]: [13/Apr/2024:07:37:31] ENGINE Serving on http://:::9283 2024-04-13T07:37:32.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:31 smithi203 bash[60651]: [13/Apr/2024:07:37:31] ENGINE Bus STARTED 2024-04-13T07:37:32.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:32 smithi203 bash[23480]: cluster 2024-04-13T07:37:31.616216+0000 mon.a (mon.0) 1103 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:37:32.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:32 smithi203 bash[23480]: cluster 2024-04-13T07:37:31.616398+0000 mon.a (mon.0) 1104 : cluster 0 Standby manager daemon x started 2024-04-13T07:37:32.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:32 smithi203 bash[23480]: audit 2024-04-13T07:37:31.622302+0000 mon.b (mon.1) 38 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:32.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:32 smithi203 bash[23480]: audit 2024-04-13T07:37:31.623041+0000 mon.b (mon.1) 39 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:32.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:32 smithi203 bash[23480]: audit 2024-04-13T07:37:31.625342+0000 mon.b (mon.1) 40 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:32.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:32 smithi203 bash[23480]: audit 2024-04-13T07:37:31.627104+0000 mon.b (mon.1) 41 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:32.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:32 smithi203 bash[23480]: audit 2024-04-13T07:37:31.852146+0000 mon.a (mon.0) 1105 : audit 0 from='client.? 172.21.15.89:0/2364380906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:32.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[31522]: cluster 2024-04-13T07:37:31.616216+0000 mon.a (mon.0) 1103 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[31522]: cluster 2024-04-13T07:37:31.616398+0000 mon.a (mon.0) 1104 : cluster 0 Standby manager daemon x started 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[31522]: audit 2024-04-13T07:37:31.622302+0000 mon.b (mon.1) 38 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[31522]: audit 2024-04-13T07:37:31.623041+0000 mon.b (mon.1) 39 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[31522]: audit 2024-04-13T07:37:31.625342+0000 mon.b (mon.1) 40 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[31522]: audit 2024-04-13T07:37:31.627104+0000 mon.b (mon.1) 41 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[31522]: audit 2024-04-13T07:37:31.852146+0000 mon.a (mon.0) 1105 : audit 0 from='client.? 172.21.15.89:0/2364380906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[21353]: cluster 2024-04-13T07:37:31.616216+0000 mon.a (mon.0) 1103 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[21353]: cluster 2024-04-13T07:37:31.616398+0000 mon.a (mon.0) 1104 : cluster 0 Standby manager daemon x started 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[21353]: audit 2024-04-13T07:37:31.622302+0000 mon.b (mon.1) 38 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[21353]: audit 2024-04-13T07:37:31.623041+0000 mon.b (mon.1) 39 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:32.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[21353]: audit 2024-04-13T07:37:31.625342+0000 mon.b (mon.1) 40 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:32.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[21353]: audit 2024-04-13T07:37:31.627104+0000 mon.b (mon.1) 41 : audit 0 from='mgr.? 172.21.15.203:0/3569679291' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:32.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:32 smithi089 bash[21353]: audit 2024-04-13T07:37:31.852146+0000 mon.a (mon.0) 1105 : audit 0 from='client.? 172.21.15.89:0/2364380906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:33.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:33 smithi203 bash[23480]: cluster 2024-04-13T07:37:32.224001+0000 mgr.y (mgr.15744) 38 : cluster 0 pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:33.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:33 smithi203 bash[23480]: cluster 2024-04-13T07:37:32.381960+0000 mon.a (mon.0) 1106 : cluster 0 mgrmap e32: y(active, since 40s), standbys: x 2024-04-13T07:37:33.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:33 smithi203 bash[60252]: ts=2024-04-13T07:37:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:37:33.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:33 smithi089 bash[21353]: cluster 2024-04-13T07:37:32.224001+0000 mgr.y (mgr.15744) 38 : cluster 0 pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:33.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:33 smithi089 bash[21353]: cluster 2024-04-13T07:37:32.381960+0000 mon.a (mon.0) 1106 : cluster 0 mgrmap e32: y(active, since 40s), standbys: x 2024-04-13T07:37:33.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:33 smithi089 bash[31522]: cluster 2024-04-13T07:37:32.224001+0000 mgr.y (mgr.15744) 38 : cluster 0 pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:33.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:33 smithi089 bash[31522]: cluster 2024-04-13T07:37:32.381960+0000 mon.a (mon.0) 1106 : cluster 0 mgrmap e32: y(active, since 40s), standbys: x 2024-04-13T07:37:34.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:34 smithi203 bash[23480]: audit 2024-04-13T07:37:34.288094+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.89:0/2125338129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:34.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:34 smithi089 bash[21353]: audit 2024-04-13T07:37:34.288094+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.89:0/2125338129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:34.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:34 smithi089 bash[31522]: audit 2024-04-13T07:37:34.288094+0000 mon.c (mon.2) 133 : audit 0 from='client.? 172.21.15.89:0/2125338129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:35.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:35 smithi203 bash[23480]: cluster 2024-04-13T07:37:34.225048+0000 mgr.y (mgr.15744) 39 : cluster 0 pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:35.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:35 smithi089 bash[21353]: cluster 2024-04-13T07:37:34.225048+0000 mgr.y (mgr.15744) 39 : cluster 0 pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:35.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:35 smithi089 bash[31522]: cluster 2024-04-13T07:37:34.225048+0000 mgr.y (mgr.15744) 39 : cluster 0 pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:36.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:37:35] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-13T07:37:37.512 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:37 smithi203 bash[23480]: cluster 2024-04-13T07:37:36.225824+0000 mgr.y (mgr.15744) 40 : cluster 0 pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:37.512 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:37 smithi203 bash[23480]: audit 2024-04-13T07:37:36.733207+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.89:0/1807160100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:37.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:37 smithi203 bash[23480]: audit 2024-04-13T07:37:37.326052+0000 mon.c (mon.2) 135 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:37.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:37 smithi089 bash[31522]: cluster 2024-04-13T07:37:36.225824+0000 mgr.y (mgr.15744) 40 : cluster 0 pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:37 smithi089 bash[31522]: audit 2024-04-13T07:37:36.733207+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.89:0/1807160100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:37 smithi089 bash[31522]: audit 2024-04-13T07:37:37.326052+0000 mon.c (mon.2) 135 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:37.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:37 smithi089 bash[21353]: cluster 2024-04-13T07:37:36.225824+0000 mgr.y (mgr.15744) 40 : cluster 0 pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:37.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:37 smithi089 bash[21353]: audit 2024-04-13T07:37:36.733207+0000 mon.c (mon.2) 134 : audit 0 from='client.? 172.21.15.89:0/1807160100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:37.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:37 smithi089 bash[21353]: audit 2024-04-13T07:37:37.326052+0000 mon.c (mon.2) 135 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:39.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:39 smithi203 bash[23480]: cluster 2024-04-13T07:37:38.226930+0000 mgr.y (mgr.15744) 41 : cluster 0 pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:39.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:39 smithi203 bash[23480]: audit 2024-04-13T07:37:39.175614+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.89:0/3106972882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:39.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:39 smithi203 bash[23480]: audit 2024-04-13T07:37:39.315285+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:39.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:39 smithi203 bash[23480]: audit 2024-04-13T07:37:39.330899+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:39.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:39 smithi203 bash[23480]: audit 2024-04-13T07:37:39.334871+0000 mon.c (mon.2) 137 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:39.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:39 smithi203 bash[23480]: audit 2024-04-13T07:37:39.336657+0000 mon.c (mon.2) 138 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:37:39.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:39 smithi203 bash[23480]: audit 2024-04-13T07:37:39.349289+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:39.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[21353]: cluster 2024-04-13T07:37:38.226930+0000 mgr.y (mgr.15744) 41 : cluster 0 pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:39.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[21353]: audit 2024-04-13T07:37:39.175614+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.89:0/3106972882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:39.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[21353]: audit 2024-04-13T07:37:39.315285+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:39.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[21353]: audit 2024-04-13T07:37:39.330899+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:39.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[21353]: audit 2024-04-13T07:37:39.334871+0000 mon.c (mon.2) 137 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:39.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[21353]: audit 2024-04-13T07:37:39.336657+0000 mon.c (mon.2) 138 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:37:39.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[21353]: audit 2024-04-13T07:37:39.349289+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:39.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[31522]: cluster 2024-04-13T07:37:38.226930+0000 mgr.y (mgr.15744) 41 : cluster 0 pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:39.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[31522]: audit 2024-04-13T07:37:39.175614+0000 mon.c (mon.2) 136 : audit 0 from='client.? 172.21.15.89:0/3106972882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:39.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[31522]: audit 2024-04-13T07:37:39.315285+0000 mon.a (mon.0) 1107 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:39.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[31522]: audit 2024-04-13T07:37:39.330899+0000 mon.a (mon.0) 1108 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:39.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[31522]: audit 2024-04-13T07:37:39.334871+0000 mon.c (mon.2) 137 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:39.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[31522]: audit 2024-04-13T07:37:39.336657+0000 mon.c (mon.2) 138 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:37:39.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:39 smithi089 bash[31522]: audit 2024-04-13T07:37:39.349289+0000 mon.a (mon.0) 1109 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:40.672 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.445572+0000 mon.c (mon.2) 139 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.451001+0000 mon.c (mon.2) 140 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: cephadm 2024-04-13T07:37:39.452505+0000 mgr.y (mgr.15744) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.463937+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.465867+0000 mon.c (mon.2) 141 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.466376+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.479409+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.482157+0000 mon.c (mon.2) 142 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.482901+0000 mon.a (mon.0) 1113 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.496037+0000 mon.a (mon.0) 1114 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.499613+0000 mon.c (mon.2) 143 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: audit 2024-04-13T07:37:39.501474+0000 mon.c (mon.2) 144 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-13T07:37:40.673 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[21353]: cephadm 2024-04-13T07:37:39.502892+0000 mgr.y (mgr.15744) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.445572+0000 mon.c (mon.2) 139 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.451001+0000 mon.c (mon.2) 140 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: cephadm 2024-04-13T07:37:39.452505+0000 mgr.y (mgr.15744) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.463937+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.465867+0000 mon.c (mon.2) 141 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.466376+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.479409+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.482157+0000 mon.c (mon.2) 142 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.482901+0000 mon.a (mon.0) 1113 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.496037+0000 mon.a (mon.0) 1114 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.499613+0000 mon.c (mon.2) 143 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:37:40.674 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: audit 2024-04-13T07:37:39.501474+0000 mon.c (mon.2) 144 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-13T07:37:40.675 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:40 smithi089 bash[31522]: cephadm 2024-04-13T07:37:39.502892+0000 mgr.y (mgr.15744) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-04-13T07:37:40.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.445572+0000 mon.c (mon.2) 139 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.451001+0000 mon.c (mon.2) 140 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: cephadm 2024-04-13T07:37:39.452505+0000 mgr.y (mgr.15744) 42 : cephadm 1 Upgrade: Setting container_image for all mgr 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.463937+0000 mon.a (mon.0) 1110 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.465867+0000 mon.c (mon.2) 141 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.466376+0000 mon.a (mon.0) 1111 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.479409+0000 mon.a (mon.0) 1112 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.482157+0000 mon.c (mon.2) 142 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.482901+0000 mon.a (mon.0) 1113 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.496037+0000 mon.a (mon.0) 1114 : audit 1 from='mgr.15744 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.499613+0000 mon.c (mon.2) 143 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: audit 2024-04-13T07:37:39.501474+0000 mon.c (mon.2) 144 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2024-04-13T07:37:40.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:40 smithi203 bash[23480]: cephadm 2024-04-13T07:37:39.502892+0000 mgr.y (mgr.15744) 43 : cephadm 1 Upgrade: It appears safe to stop mon.a 2024-04-13T07:37:41.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:41 smithi203 bash[23480]: cluster 2024-04-13T07:37:40.227655+0000 mgr.y (mgr.15744) 44 : cluster 0 pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:41.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:41 smithi203 bash[23480]: audit 2024-04-13T07:37:41.477876+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:41.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:41 smithi203 bash[23480]: audit 2024-04-13T07:37:41.483467+0000 mon.c (mon.2) 145 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:37:41.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:41 smithi203 bash[23480]: audit 2024-04-13T07:37:41.484866+0000 mon.c (mon.2) 146 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:37:41.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:41 smithi203 bash[23480]: audit 2024-04-13T07:37:41.486370+0000 mon.c (mon.2) 147 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:41.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[31522]: cluster 2024-04-13T07:37:40.227655+0000 mgr.y (mgr.15744) 44 : cluster 0 pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:41.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[31522]: audit 2024-04-13T07:37:41.477876+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:41.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[31522]: audit 2024-04-13T07:37:41.483467+0000 mon.c (mon.2) 145 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:37:41.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[31522]: audit 2024-04-13T07:37:41.484866+0000 mon.c (mon.2) 146 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:37:41.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[31522]: audit 2024-04-13T07:37:41.486370+0000 mon.c (mon.2) 147 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:41.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[21353]: cluster 2024-04-13T07:37:40.227655+0000 mgr.y (mgr.15744) 44 : cluster 0 pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:41.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[21353]: audit 2024-04-13T07:37:41.477876+0000 mon.a (mon.0) 1115 : audit 1 from='mgr.15744 ' entity='mgr.y' 2024-04-13T07:37:41.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[21353]: audit 2024-04-13T07:37:41.483467+0000 mon.c (mon.2) 145 : audit 1 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:37:41.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[21353]: audit 2024-04-13T07:37:41.484866+0000 mon.c (mon.2) 146 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:37:41.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:41 smithi089 bash[21353]: audit 2024-04-13T07:37:41.486370+0000 mon.c (mon.2) 147 : audit 0 from='mgr.15744 172.21.15.89:0/4247477414' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:37:42.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:42 smithi203 bash[23480]: cephadm 2024-04-13T07:37:41.471621+0000 mgr.y (mgr.15744) 45 : cephadm 1 Upgrade: Updating mon.a 2024-04-13T07:37:42.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:42 smithi203 bash[23480]: cephadm 2024-04-13T07:37:41.487983+0000 mgr.y (mgr.15744) 46 : cephadm 1 Deploying daemon mon.a on smithi089 2024-04-13T07:37:42.763 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:42 smithi203 bash[23480]: audit 2024-04-13T07:37:41.641105+0000 mon.a (mon.0) 1116 : audit 0 from='client.? 172.21.15.89:0/153916144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:42.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:42 smithi203 bash[60252]: ts=2024-04-13T07:37:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:37:42.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:42 smithi089 bash[31522]: cephadm 2024-04-13T07:37:41.471621+0000 mgr.y (mgr.15744) 45 : cephadm 1 Upgrade: Updating mon.a 2024-04-13T07:37:42.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:42 smithi089 bash[31522]: cephadm 2024-04-13T07:37:41.487983+0000 mgr.y (mgr.15744) 46 : cephadm 1 Deploying daemon mon.a on smithi089 2024-04-13T07:37:42.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:42 smithi089 bash[31522]: audit 2024-04-13T07:37:41.641105+0000 mon.a (mon.0) 1116 : audit 0 from='client.? 172.21.15.89:0/153916144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:42.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:42 smithi089 bash[21353]: cephadm 2024-04-13T07:37:41.471621+0000 mgr.y (mgr.15744) 45 : cephadm 1 Upgrade: Updating mon.a 2024-04-13T07:37:42.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:42 smithi089 bash[21353]: cephadm 2024-04-13T07:37:41.487983+0000 mgr.y (mgr.15744) 46 : cephadm 1 Deploying daemon mon.a on smithi089 2024-04-13T07:37:42.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:42 smithi089 bash[21353]: audit 2024-04-13T07:37:41.641105+0000 mon.a (mon.0) 1116 : audit 0 from='client.? 172.21.15.89:0/153916144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:43.706 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.707 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:43 smithi089 bash[31522]: cluster 2024-04-13T07:37:42.228333+0000 mgr.y (mgr.15744) 47 : cluster 0 pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:43.707 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.707 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.708 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.708 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.708 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.708 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.709 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:43 smithi089 bash[21353]: cluster 2024-04-13T07:37:42.228333+0000 mgr.y (mgr.15744) 47 : cluster 0 pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:43.709 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.709 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:43.762 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:43 smithi203 bash[23480]: cluster 2024-04-13T07:37:42.228333+0000 mgr.y (mgr.15744) 47 : cluster 0 pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:43.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:43 smithi203 bash[60252]: ts=2024-04-13T07:37:43.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:37:43.979 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:43 smithi089 systemd[1]: Stopping Ceph mon.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:37:43.979 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:43 smithi089 bash[21353]: debug 2024-04-13T07:37:43.800+0000 7f1450051700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-13T07:37:43.979 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:43 smithi089 bash[21353]: debug 2024-04-13T07:37:43.800+0000 7f1450051700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2024-04-13T07:37:44.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:44 smithi089 bash[82189]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mon-a 2024-04-13T07:37:44.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:44 smithi089 bash[82275]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mon-a 2024-04-13T07:37:44.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a.service: Deactivated successfully. 2024-04-13T07:37:44.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: Stopped Ceph mon.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:37:45.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.049 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.050 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.050 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.050 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.051 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: Started Ceph mon.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:37:45.051 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:37:44 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:37:45.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.520+0000 7f96146e4c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-04-13T07:37:45.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.520+0000 7f96146e4c80 0 ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev), process ceph-mon, pid 7 2024-04-13T07:37:45.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.520+0000 7f96146e4c80 0 pidfile_write: ignore empty --pid-file 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 0 load: jerasure load: lrc 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: RocksDB version: 7.9.2 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Git sha 0 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Compile date 2024-04-12 15:38:20 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: DB SUMMARY 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: DB Session ID: ALQZE2KPX91KJK7XCPP7 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: CURRENT file: CURRENT 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: IDENTITY file: IDENTITY 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: MANIFEST file: MANIFEST-000015 size: 1729 Bytes 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000035.sst 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000033.log size: 1105 ; 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.error_if_exists: 0 2024-04-13T07:37:45.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.create_if_missing: 0 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.paranoid_checks: 1 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.env: 0x55bbe5b0fcc0 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.fs: PosixFileSystem 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.info_log: 0x55bbe7567c40 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.statistics: (nil) 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.use_fsync: 0 2024-04-13T07:37:45.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_log_file_size: 0 2024-04-13T07:37:45.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-13T07:37:45.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-04-13T07:37:45.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-04-13T07:37:45.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-04-13T07:37:45.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.allow_fallocate: 1 2024-04-13T07:37:45.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.use_direct_reads: 0 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.create_missing_column_families: 0 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.db_log_dir: 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.wal_dir: 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-13T07:37:45.803 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.advise_random_on_open: 1 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.write_buffer_manager: 0x55bbe753a820 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.rate_limiter: (nil) 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.unordered_write: 0 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-13T07:37:45.804 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.row_cache: None 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.wal_filter: None 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.two_write_queues: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.manual_wal_flush: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.wal_compression: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.atomic_flush: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-04-13T07:37:45.805 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.log_readahead_size: 0 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_background_jobs: 2 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_background_compactions: -1 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_subcompactions: 1 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-13T07:37:45.806 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_total_wal_size: 0 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_open_files: -1 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bytes_per_sync: 0 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_background_flushes: -1 2024-04-13T07:37:45.807 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Compression algorithms supported: 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: kZSTD supported: 0 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: kXpressCompression supported: 0 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: kZlibCompression supported: 1 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: kSnappyCompression supported: 1 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: kLZ4Compression supported: 1 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: kBZip2Compression supported: 0 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-04-13T07:37:45.808 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.merge_operator: 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_filter: None 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_filter_factory: None 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55bbe7567e00) 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: cache_index_and_filter_blocks: 1 2024-04-13T07:37:45.810 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: pin_top_level_index_and_filter: 1 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: index_type: 0 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: data_block_index_type: 0 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: index_shortening: 1 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: data_block_hash_table_util_ratio: 0.750000 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: checksum: 4 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: no_block_cache: 0 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: block_cache: 0x55bbe756b090 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: block_cache_name: BinnedLRUCache 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: block_cache_options: 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: capacity : 536870912 2024-04-13T07:37:45.811 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: num_shard_bits : 4 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: strict_capacity_limit : 0 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: high_pri_pool_ratio: 0.000 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: block_cache_compressed: (nil) 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: persistent_cache: (nil) 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: block_size: 4096 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: block_size_deviation: 10 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: block_restart_interval: 16 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: index_block_restart_interval: 1 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: metadata_block_size: 4096 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: partition_filters: 0 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: use_delta_encoding: 1 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: filter_policy: bloomfilter 2024-04-13T07:37:45.812 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: whole_key_filtering: 1 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: verify_compression: 0 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: read_amp_bytes_per_bit: 0 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: format_version: 5 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: enable_index_compression: 1 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: block_align: 0 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: max_auto_readahead_size: 262144 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: prepopulate_block_cache: 0 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: initial_auto_readahead_size: 8192 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: num_file_reads_for_auto_readahead: 2 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression: NoCompression 2024-04-13T07:37:45.813 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.num_levels: 7 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-13T07:37:45.814 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.level: 32767 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-04-13T07:37:45.815 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.enabled: false 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-13T07:37:45.816 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.arena_block_size: 1048576 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-13T07:37:45.817 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.inplace_update_support: 0 2024-04-13T07:37:45.818 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.bloom_locality: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.max_successive_merges: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.force_consistency_checks: 1 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.ttl: 2592000 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.enable_blob_files: false 2024-04-13T07:37:45.819 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.min_blob_size: 0 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.blob_file_size: 268435456 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 37, last_sequence is 15987, log_number is 33,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 33 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 33 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: ce66e667-534b-4b0f-bd19-3c033b687d2c 2024-04-13T07:37:45.820 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993865530644, "job": 1, "event": "recovery_started", "wal_files": [33]} 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.524+0000 7f96146e4c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #33 mode 2 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993865531155, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 38, "file_size": 2228, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 15988, "largest_seqno": 15994, "table_properties": {"data_size": 1100, "index_size": 33, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 165, "raw_average_key_size": 23, "raw_value_size": 934, "raw_average_value_size": 133, "num_data_blocks": 1, "num_entries": 7, "num_filter_entries": 7, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712993865, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "ce66e667-534b-4b0f-bd19-3c033b687d2c", "db_session_id": "ALQZE2KPX91KJK7XCPP7", "orig_file_number": 38, "seqno_to_time_mapping": "N/A"}} 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993865531257, "job": 1, "event": "recovery_finished"} 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 40 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000033.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55bbe765e000 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 4 rocksdb: DB pointer 0x55bbe7648000 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f9602e3e700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f9602e3e700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: ** DB Stats ** 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-13T07:37:45.821 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: ** Compaction Stats [default] ** 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: L0 1/0 2.18 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: L6 1/0 7.74 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Sum 2/0 7.74 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 4.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: ** Compaction Stats [default] ** 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:37:45.822 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 4.4 0.00 0.00 1 0.000 0 0 0.0 0.0 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Flush(GB): cumulative 0.000, interval 0.000 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: AddFile(Total Files): cumulative 0, interval 0 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: AddFile(L0 Files): cumulative 0, interval 0 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: AddFile(Keys): cumulative 0, interval 0 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Cumulative compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Interval compaction: 0.00 GB write, 0.62 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Block cache BinnedLRUCache@0x55bbe756b090#7 capacity: 512.00 MB usage: 0.23 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Block cache entry stats(count,size,portion): FilterBlock(1,0.11 KB,2.08616e-05%) IndexBlock(1,0.12 KB,2.38419e-05%) Misc(1,0.00 KB,0%) 2024-04-13T07:37:45.823 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: ** File Read Latency Histogram By Level [default] ** 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 0 starting mon.a rank 0 at public addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] at bind addrs [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 1 mon.a@-1(???) e3 preinit fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 0 mon.a@-1(???).mds e10 new map 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.528+0000 7f96146e4c80 0 mon.a@-1(???).mds e10 print_map 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: e10 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: enable_multiple, ever_enabled_multiple: 1,1 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: legacy client fscid: 1 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Filesystem 'foo' (1) 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: fs_name foo 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: epoch 10 2024-04-13T07:37:45.824 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: created 2024-04-13T07:32:03.054183+0000 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: modified 2024-04-13T07:32:13.856309+0000 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: tableserver 0 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: root 0 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: session_timeout 60 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: session_autoclose 300 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: max_file_size 1099511627776 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: max_xattr_size 65536 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: required_client_features {} 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: last_failure 0 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: last_failure_osd_epoch 70 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: max_mds 1 2024-04-13T07:37:45.825 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: in 0 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: up {0=24364} 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: failed 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: damaged 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: stopped 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: data_pools [4] 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: metadata_pool 3 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: inline_data disabled 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: balancer 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: bal_rank_mask -1 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: standby_count_wanted 1 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: qdb_cluster leader: 0 members: 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: [mds.foo.smithi203.sjspdy{0:24364} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] compat {c=[1],r=[1],i=[7ff]}] 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: 2024-04-13T07:37:45.826 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: 2024-04-13T07:37:45.827 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: Standby daemons: 2024-04-13T07:37:45.827 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: 2024-04-13T07:37:45.827 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: [mds.foo.smithi089.pccnvq{-1:14583} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.89:6834/2537258916,v1:172.21.15.89:6835/2537258916] compat {c=[1],r=[1],i=[7ff]}] 2024-04-13T07:37:45.827 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.532+0000 7f96146e4c80 0 mon.a@-1(???).osd e91 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-13T07:37:45.827 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.532+0000 7f96146e4c80 0 mon.a@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:37:45.827 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.532+0000 7f96146e4c80 0 mon.a@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:37:45.827 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.532+0000 7f96146e4c80 0 mon.a@-1(???).osd e91 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:37:45.827 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[82331]: debug 2024-04-13T07:37:45.532+0000 7f96146e4c80 1 mon.a@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2024-04-13T07:37:46.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:37:45] "GET /metrics HTTP/1.1" 200 33825 "" "Prometheus/2.43.0" 2024-04-13T07:37:49.001 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev)": 1 2024-04-13T07:37:49.002 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[74725]: ignoring --setuser ceph since I am not root 2024-04-13T07:37:49.002 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[74725]: ignoring --setgroup ceph since I am not root 2024-04-13T07:37:49.002 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[74725]: debug 2024-04-13T07:37:48.692+0000 7fab1a302700 1 -- 172.21.15.89:0/2110598937 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x564929ca45a0 con 0x56492aa22400 2024-04-13T07:37:49.002 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[74725]: debug 2024-04-13T07:37:48.840+0000 7fab240f4200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: audit 2024-04-13T07:37:44.091987+0000 mon.b (mon.1) 42 : audit [DBG] from='client.15951 172.21.15.89:0/2367677891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:44.229485+0000 mgr.y (mgr.15744) 48 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:46.230008+0000 mgr.y (mgr.15744) 49 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: audit 2024-04-13T07:37:46.538532+0000 mon.c (mon.2) 148 : audit [DBG] from='client.? 172.21.15.89:0/64680055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:46.956973+0000 mon.a (mon.0) 1 : cluster [INF] mon.a calling monitor election 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.582418+0000 mon.a (mon.0) 2 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591551+0000 mon.a (mon.0) 3 : cluster [DBG] monmap epoch 3 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591582+0000 mon.a (mon.0) 4 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591598+0000 mon.a (mon.0) 5 : cluster [DBG] last_changed 2024-04-13T07:21:29.816921+0000 2024-04-13T07:37:49.003 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591616+0000 mon.a (mon.0) 6 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591632+0000 mon.a (mon.0) 7 : cluster [DBG] min_mon_release 18 (reef) 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591648+0000 mon.a (mon.0) 8 : cluster [DBG] election_strategy: 1 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591668+0000 mon.a (mon.0) 9 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591689+0000 mon.a (mon.0) 10 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.591707+0000 mon.a (mon.0) 11 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.592699+0000 mon.a (mon.0) 12 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.592748+0000 mon.a (mon.0) 13 : cluster [DBG] osdmap e91: 8 total, 8 up, 8 in 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.594643+0000 mon.a (mon.0) 14 : cluster [DBG] mgrmap e32: y(active, since 56s), standbys: x 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.595521+0000 mon.a (mon.0) 15 : cluster [INF] overall HEALTH_OK 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: audit 2024-04-13T07:37:48.622658+0000 mon.a (mon.0) 16 : audit [INF] from='mgr.15744 ' entity='' 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[82331]: cluster 2024-04-13T07:37:48.623952+0000 mon.a (mon.0) 17 : cluster [DBG] mgrmap e33: y(active, since 56s), standbys: x 2024-04-13T07:37:49.004 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: audit 2024-04-13T07:37:44.091987+0000 mon.b (mon.1) 42 : audit 0 from='client.15951 172.21.15.89:0/2367677891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:44.229485+0000 mgr.y (mgr.15744) 48 : cluster 0 pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:46.230008+0000 mgr.y (mgr.15744) 49 : cluster 0 pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: audit 2024-04-13T07:37:46.538532+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.89:0/64680055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:46.956973+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.582418+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591551+0000 mon.a (mon.0) 3 : cluster 0 monmap epoch 3 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591582+0000 mon.a (mon.0) 4 : cluster 0 fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591598+0000 mon.a (mon.0) 5 : cluster 0 last_changed 2024-04-13T07:21:29.816921+0000 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591616+0000 mon.a (mon.0) 6 : cluster 0 created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591632+0000 mon.a (mon.0) 7 : cluster 0 min_mon_release 18 (reef) 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591648+0000 mon.a (mon.0) 8 : cluster 0 election_strategy: 1 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591668+0000 mon.a (mon.0) 9 : cluster 0 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:37:49.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591689+0000 mon.a (mon.0) 10 : cluster 0 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:37:49.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.591707+0000 mon.a (mon.0) 11 : cluster 0 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:37:49.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.592699+0000 mon.a (mon.0) 12 : cluster 0 fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:37:49.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.592748+0000 mon.a (mon.0) 13 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-04-13T07:37:49.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.594643+0000 mon.a (mon.0) 14 : cluster 0 mgrmap e32: y(active, since 56s), standbys: x 2024-04-13T07:37:49.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.595521+0000 mon.a (mon.0) 15 : cluster 1 overall HEALTH_OK 2024-04-13T07:37:49.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: audit 2024-04-13T07:37:48.622658+0000 mon.a (mon.0) 16 : audit 1 from='mgr.15744 ' entity='' 2024-04-13T07:37:49.006 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:48 smithi089 bash[31522]: cluster 2024-04-13T07:37:48.623952+0000 mon.a (mon.0) 17 : cluster 0 mgrmap e33: y(active, since 56s), standbys: x 2024-04-13T07:37:49.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[60651]: ignoring --setuser ceph since I am not root 2024-04-13T07:37:49.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[60651]: ignoring --setgroup ceph since I am not root 2024-04-13T07:37:49.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[60651]: debug 2024-04-13T07:37:48.715+0000 7f405d5b3700 1 -- 172.21.15.203:0/190835377 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x5621572e25a0 con 0x562158060400 2024-04-13T07:37:49.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[60651]: debug 2024-04-13T07:37:48.859+0000 7f40673a5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: audit 2024-04-13T07:37:44.091987+0000 mon.b (mon.1) 42 : audit 0 from='client.15951 172.21.15.89:0/2367677891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:44.229485+0000 mgr.y (mgr.15744) 48 : cluster 0 pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:46.230008+0000 mgr.y (mgr.15744) 49 : cluster 0 pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: audit 2024-04-13T07:37:46.538532+0000 mon.c (mon.2) 148 : audit 0 from='client.? 172.21.15.89:0/64680055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:46.956973+0000 mon.a (mon.0) 1 : cluster 1 mon.a calling monitor election 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.582418+0000 mon.a (mon.0) 2 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591551+0000 mon.a (mon.0) 3 : cluster 0 monmap epoch 3 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591582+0000 mon.a (mon.0) 4 : cluster 0 fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591598+0000 mon.a (mon.0) 5 : cluster 0 last_changed 2024-04-13T07:21:29.816921+0000 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591616+0000 mon.a (mon.0) 6 : cluster 0 created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591632+0000 mon.a (mon.0) 7 : cluster 0 min_mon_release 18 (reef) 2024-04-13T07:37:49.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591648+0000 mon.a (mon.0) 8 : cluster 0 election_strategy: 1 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591668+0000 mon.a (mon.0) 9 : cluster 0 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591689+0000 mon.a (mon.0) 10 : cluster 0 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.591707+0000 mon.a (mon.0) 11 : cluster 0 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.592699+0000 mon.a (mon.0) 12 : cluster 0 fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.592748+0000 mon.a (mon.0) 13 : cluster 0 osdmap e91: 8 total, 8 up, 8 in 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.594643+0000 mon.a (mon.0) 14 : cluster 0 mgrmap e32: y(active, since 56s), standbys: x 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.595521+0000 mon.a (mon.0) 15 : cluster 1 overall HEALTH_OK 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: audit 2024-04-13T07:37:48.622658+0000 mon.a (mon.0) 16 : audit 1 from='mgr.15744 ' entity='' 2024-04-13T07:37:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:48 smithi203 bash[23480]: cluster 2024-04-13T07:37:48.623952+0000 mon.a (mon.0) 17 : cluster 0 mgrmap e33: y(active, since 56s), standbys: x 2024-04-13T07:37:49.775 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch ps' 2024-04-13T07:37:50.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:49 smithi203 bash[23480]: audit 2024-04-13T07:37:48.994201+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.89:0/2176710314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:50.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:49 smithi089 bash[82331]: audit 2024-04-13T07:37:48.994201+0000 mon.c (mon.2) 149 : audit [DBG] from='client.? 172.21.15.89:0/2176710314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:50.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:49 smithi089 bash[31522]: audit 2024-04-13T07:37:48.994201+0000 mon.c (mon.2) 149 : audit 0 from='client.? 172.21.15.89:0/2176710314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:37:50.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:50 smithi203 bash[60651]: debug 2024-04-13T07:37:50.435+0000 7f40673a5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:37:50.762 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:50 smithi203 bash[60651]: debug 2024-04-13T07:37:50.531+0000 7f40673a5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:37:50.796 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:50 smithi089 bash[74725]: debug 2024-04-13T07:37:50.448+0000 7fab240f4200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2024-04-13T07:37:50.796 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:50 smithi089 bash[74725]: debug 2024-04-13T07:37:50.548+0000 7fab240f4200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2024-04-13T07:37:51.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:50 smithi203 bash[60651]: debug 2024-04-13T07:37:50.783+0000 7f40673a5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:37:52.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:51 smithi203 bash[60651]: debug 2024-04-13T07:37:51.583+0000 7f40673a5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:37:52.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:51 smithi203 bash[60651]: debug 2024-04-13T07:37:51.695+0000 7f40673a5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:37:52.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:51 smithi203 bash[60651]: debug 2024-04-13T07:37:51.803+0000 7f40673a5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:37:52.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:50 smithi089 bash[74725]: debug 2024-04-13T07:37:50.796+0000 7fab240f4200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2024-04-13T07:37:52.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:51 smithi089 bash[74725]: debug 2024-04-13T07:37:51.600+0000 7fab240f4200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2024-04-13T07:37:52.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:51 smithi089 bash[74725]: debug 2024-04-13T07:37:51.716+0000 7fab240f4200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2024-04-13T07:37:52.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:51 smithi089 bash[74725]: debug 2024-04-13T07:37:51.820+0000 7fab240f4200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2024-04-13T07:37:52.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:52 smithi203 bash[60651]: debug 2024-04-13T07:37:52.167+0000 7f40673a5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:37:52.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:52 smithi203 bash[60651]: debug 2024-04-13T07:37:52.283+0000 7f40673a5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:37:52.513 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:52 smithi203 bash[60651]: debug 2024-04-13T07:37:52.391+0000 7f40673a5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:37:52.513 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:52 smithi203 bash[60252]: ts=2024-04-13T07:37:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:37:52.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:52 smithi089 bash[74725]: debug 2024-04-13T07:37:52.184+0000 7fab240f4200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2024-04-13T07:37:52.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:52 smithi089 bash[74725]: debug 2024-04-13T07:37:52.300+0000 7fab240f4200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2024-04-13T07:37:52.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:52 smithi089 bash[74725]: debug 2024-04-13T07:37:52.408+0000 7fab240f4200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2024-04-13T07:37:53.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:52 smithi203 bash[60651]: debug 2024-04-13T07:37:52.751+0000 7f40673a5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:37:53.013 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:53 smithi203 bash[60651]: debug 2024-04-13T07:37:52.995+0000 7f40673a5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:37:53.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:52 smithi089 bash[74725]: debug 2024-04-13T07:37:52.772+0000 7fab240f4200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2024-04-13T07:37:53.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:53 smithi089 bash[74725]: debug 2024-04-13T07:37:53.016+0000 7fab240f4200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2024-04-13T07:37:53.512 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:37:53 smithi203 bash[60252]: ts=2024-04-13T07:37:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:37:53.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:53 smithi203 bash[60651]: debug 2024-04-13T07:37:53.263+0000 7f40673a5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:37:53.513 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:53 smithi203 bash[60651]: debug 2024-04-13T07:37:53.379+0000 7f40673a5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:37:53.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:53 smithi089 bash[74725]: debug 2024-04-13T07:37:53.284+0000 7fab240f4200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2024-04-13T07:37:53.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:53 smithi089 bash[74725]: debug 2024-04-13T07:37:53.400+0000 7fab240f4200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2024-04-13T07:37:54.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:53 smithi203 bash[60651]: debug 2024-04-13T07:37:53.655+0000 7f40673a5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:37:54.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:53 smithi089 bash[74725]: debug 2024-04-13T07:37:53.680+0000 7fab240f4200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2024-04-13T07:37:54.438 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:54 smithi203 bash[60651]: debug 2024-04-13T07:37:54.079+0000 7f40673a5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:37:54.438 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:54 smithi203 bash[60651]: debug 2024-04-13T07:37:54.319+0000 7f40673a5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:37:54.468 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:54 smithi089 bash[74725]: debug 2024-04-13T07:37:54.104+0000 7fab240f4200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:37:54.469 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:54 smithi089 bash[74725]: debug 2024-04-13T07:37:54.348+0000 7fab240f4200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2024-04-13T07:37:54.482 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:37:54.694 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:54 smithi203 bash[60651]: debug 2024-04-13T07:37:54.443+0000 7f40673a5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:37:54.726 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:54 smithi089 bash[74725]: debug 2024-04-13T07:37:54.472+0000 7fab240f4200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2024-04-13T07:37:55.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:54 smithi203 bash[60651]: debug 2024-04-13T07:37:54.699+0000 7f40673a5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:37:55.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:54 smithi089 bash[74725]: debug 2024-04-13T07:37:54.728+0000 7fab240f4200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2024-04-13T07:37:56.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:55 smithi203 bash[60651]: debug 2024-04-13T07:37:55.731+0000 7f40673a5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:37:56.012 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:55 smithi203 bash[60651]: debug 2024-04-13T07:37:55.851+0000 7f40673a5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:37:56.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:55 smithi089 bash[74725]: debug 2024-04-13T07:37:55.780+0000 7fab240f4200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2024-04-13T07:37:56.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:55 smithi089 bash[74725]: debug 2024-04-13T07:37:55.900+0000 7fab240f4200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2024-04-13T07:37:56.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: debug 2024-04-13T07:37:56.151+0000 7f40673a5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:37:56.512 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: debug 2024-04-13T07:37:56.267+0000 7f40673a5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:37:56.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[74725]: debug 2024-04-13T07:37:56.208+0000 7fab240f4200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2024-04-13T07:37:56.548 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[74725]: debug 2024-04-13T07:37:56.328+0000 7fab240f4200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2024-04-13T07:37:56.985 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: debug 2024-04-13T07:37:56.731+0000 7f40673a5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:37:56.985 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: debug 2024-04-13T07:37:56.859+0000 7f40673a5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:37:56.985 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: [13/Apr/2024:07:37:56] ENGINE Bus STARTING 2024-04-13T07:37:56.985 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[23480]: cluster 2024-04-13T07:37:56.883889+0000 mon.a (mon.0) 18 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:37:56.985 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[23480]: cluster 2024-04-13T07:37:56.884045+0000 mon.a (mon.0) 19 : cluster 0 Standby manager daemon x started 2024-04-13T07:37:56.985 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[23480]: audit 2024-04-13T07:37:56.888752+0000 mon.b (mon.1) 43 : audit 0 from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:56.985 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[23480]: audit 2024-04-13T07:37:56.889431+0000 mon.b (mon.1) 44 : audit 0 from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:56.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[23480]: audit 2024-04-13T07:37:56.891554+0000 mon.b (mon.1) 45 : audit 0 from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:56.986 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[23480]: audit 2024-04-13T07:37:56.893352+0000 mon.b (mon.1) 46 : audit 0 from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:57.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[74725]: debug 2024-04-13T07:37:56.796+0000 7fab240f4200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2024-04-13T07:37:57.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[74725]: debug 2024-04-13T07:37:56.928+0000 7fab240f4200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2024-04-13T07:37:57.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[31522]: cluster 2024-04-13T07:37:56.883889+0000 mon.a (mon.0) 18 : cluster 0 Standby manager daemon x restarted 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[31522]: cluster 2024-04-13T07:37:56.884045+0000 mon.a (mon.0) 19 : cluster 0 Standby manager daemon x started 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[31522]: audit 2024-04-13T07:37:56.888752+0000 mon.b (mon.1) 43 : audit 0 from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[31522]: audit 2024-04-13T07:37:56.889431+0000 mon.b (mon.1) 44 : audit 0 from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[31522]: audit 2024-04-13T07:37:56.891554+0000 mon.b (mon.1) 45 : audit 0 from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[31522]: audit 2024-04-13T07:37:56.893352+0000 mon.b (mon.1) 46 : audit 0 from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[82331]: debug 2024-04-13T07:37:56.928+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[82331]: cluster 2024-04-13T07:37:56.883889+0000 mon.a (mon.0) 18 : cluster [DBG] Standby manager daemon x restarted 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[82331]: cluster 2024-04-13T07:37:56.884045+0000 mon.a (mon.0) 19 : cluster [DBG] Standby manager daemon x started 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[82331]: audit 2024-04-13T07:37:56.888752+0000 mon.b (mon.1) 43 : audit [DBG] from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2024-04-13T07:37:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[82331]: audit 2024-04-13T07:37:56.889431+0000 mon.b (mon.1) 44 : audit [DBG] from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2024-04-13T07:37:57.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[82331]: audit 2024-04-13T07:37:56.891554+0000 mon.b (mon.1) 45 : audit [DBG] from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2024-04-13T07:37:57.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:56 smithi089 bash[82331]: audit 2024-04-13T07:37:56.893352+0000 mon.b (mon.1) 46 : audit [DBG] from='mgr.? 172.21.15.203:0/4226768395' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2024-04-13T07:37:57.262 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: CherryPy Checker: 2024-04-13T07:37:57.263 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: The Application mounted at '' has an empty config. 2024-04-13T07:37:57.263 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: [13/Apr/2024:07:37:56] ENGINE Serving on http://:::9283 2024-04-13T07:37:57.263 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:37:56 smithi203 bash[60651]: [13/Apr/2024:07:37:56] ENGINE Bus STARTED 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: [13/Apr/2024:07:37:57] ENGINE Bus STARTING 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa89419700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa89419700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa89419700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa89419700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa89419700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa81c0a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa81c0a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa81c0a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.549 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa81c0a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.550 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.124+0000 7faa81c0a700 -1 client.0 error registering admin socket command: (17) File exists 2024-04-13T07:37:57.550 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: CherryPy Checker: 2024-04-13T07:37:57.550 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: The Application mounted at '' has an empty config. 2024-04-13T07:37:57.550 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: [13/Apr/2024:07:37:57] ENGINE Serving on http://:::9283 2024-04-13T07:37:57.550 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: [13/Apr/2024:07:37:57] ENGINE Bus STARTED 2024-04-13T07:37:57.989 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-13T07:37:57.989 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (7m) 59s ago 8m 13.8M - 0.25.0 c8568f914cd2 d39c47986efe 2024-04-13T07:37:57.989 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi203 *:3000 running (7m) 29s ago 7m 78.2M - 9.4.7 954c08fa6188 4496ec520de5 2024-04-13T07:37:57.989 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.pccnvq smithi089 running (5m) 59s ago 5m 13.2M - 18.2.2-1217-g7d36322c 6ecd1226aa2f 07167bfc6e5f 2024-04-13T07:37:57.989 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi203.sjspdy smithi203 running (5m) 29s ago 5m 15.2M - 18.2.2-1217-g7d36322c 6ecd1226aa2f e100a52f8447 2024-04-13T07:37:57.989 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi203 *:8443,9283,8765 running (35s) 29s ago 16m 113M - 19.0.0-2905-g5f22a497 c4b45bf0c946 54a5a6c61bc7 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (82s) 59s ago 18m 467M - 19.0.0-2905-g5f22a497 c4b45bf0c946 f20e719fbf68 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (18m) 59s ago 18m 53.8M 2048M 18.2.2-1217-g7d36322c 6ecd1226aa2f 835fb959f8c6 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi203 running (16m) 29s ago 16m 46.2M 2048M 18.2.2-1217-g7d36322c 6ecd1226aa2f e475d5b92fda 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (16m) 59s ago 16m 44.0M 2048M 18.2.2-1217-g7d36322c 6ecd1226aa2f 2b26393a116d 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (8m) 59s ago 8m 8231k - 1.5.0 0da6a335fe13 04462903d931 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi203 *:9100 running (8m) 29s ago 8m 8479k - 1.5.0 0da6a335fe13 a79084a2b37c 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (15m) 59s ago 15m 60.0M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 599c6c3ff3d6 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (14m) 59s ago 14m 51.3M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 58fb3beb85cc 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (13m) 59s ago 13m 50.1M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f e020a590fd83 2024-04-13T07:37:57.990 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (12m) 59s ago 12m 46.2M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 1c6772886276 2024-04-13T07:37:57.991 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi203 running (12m) 29s ago 12m 54.7M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 28af48f1a163 2024-04-13T07:37:57.991 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi203 running (11m) 29s ago 11m 47.8M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f afe79fa4d7d0 2024-04-13T07:37:57.991 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi203 running (10m) 29s ago 10m 46.5M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 8913a43d05b8 2024-04-13T07:37:57.993 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi203 running (9m) 29s ago 9m 61.2M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 01ed33979f80 2024-04-13T07:37:57.994 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi203 *:9095 running (42s) 29s ago 8m 33.6M - 2.43.0 a07b618ecd1d 50a6268920e0 2024-04-13T07:37:58.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: cluster 2024-04-13T07:37:56.933802+0000 mon.a (mon.0) 20 : cluster 1 Active manager daemon y restarted 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: cluster 2024-04-13T07:37:56.934332+0000 mon.a (mon.0) 21 : cluster 4 Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: cluster 2024-04-13T07:37:56.934682+0000 mon.a (mon.0) 22 : cluster 1 Activating manager daemon y 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: cluster 2024-04-13T07:37:56.941844+0000 mon.a (mon.0) 23 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: cluster 2024-04-13T07:37:56.942365+0000 mon.a (mon.0) 24 : cluster 0 mgrmap e34: y(active, starting, since 0.00795721s), standbys: x 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.951253+0000 mon.a (mon.0) 25 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.951632+0000 mon.a (mon.0) 26 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.952065+0000 mon.a (mon.0) 27 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.956573+0000 mon.a (mon.0) 28 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.957167+0000 mon.a (mon.0) 29 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.957625+0000 mon.a (mon.0) 30 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.958001+0000 mon.a (mon.0) 31 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.958630+0000 mon.a (mon.0) 32 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:37:58.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.959317+0000 mon.a (mon.0) 33 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.959874+0000 mon.a (mon.0) 34 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.960375+0000 mon.a (mon.0) 35 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.960854+0000 mon.a (mon.0) 36 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.961350+0000 mon.a (mon.0) 37 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.961838+0000 mon.a (mon.0) 38 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.962351+0000 mon.a (mon.0) 39 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.962862+0000 mon.a (mon.0) 40 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.963429+0000 mon.a (mon.0) 41 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:56.964625+0000 mon.a (mon.0) 42 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: cluster 2024-04-13T07:37:56.973192+0000 mon.a (mon.0) 43 : cluster 1 Manager daemon y is now available 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:57.015505+0000 mon.a (mon.0) 44 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:57.059085+0000 mon.a (mon.0) 45 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:37:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:57.060703+0000 mon.a (mon.0) 46 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:37:58.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:57.063835+0000 mon.a (mon.0) 47 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:58.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:57 smithi203 bash[23480]: audit 2024-04-13T07:37:57.090481+0000 mon.a (mon.0) 48 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:37:58.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[74725]: debug 2024-04-13T07:37:57.976+0000 7faae2853700 -1 mgr.server handle_report got status from non-daemon mon.a 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: cluster 2024-04-13T07:37:56.933802+0000 mon.a (mon.0) 20 : cluster [INF] Active manager daemon y restarted 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: cluster 2024-04-13T07:37:56.934332+0000 mon.a (mon.0) 21 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: cluster 2024-04-13T07:37:56.934682+0000 mon.a (mon.0) 22 : cluster [INF] Activating manager daemon y 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: cluster 2024-04-13T07:37:56.941844+0000 mon.a (mon.0) 23 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: cluster 2024-04-13T07:37:56.942365+0000 mon.a (mon.0) 24 : cluster [DBG] mgrmap e34: y(active, starting, since 0.00795721s), standbys: x 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.951253+0000 mon.a (mon.0) 25 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.951632+0000 mon.a (mon.0) 26 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.952065+0000 mon.a (mon.0) 27 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:37:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.956573+0000 mon.a (mon.0) 28 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.957167+0000 mon.a (mon.0) 29 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.957625+0000 mon.a (mon.0) 30 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.958001+0000 mon.a (mon.0) 31 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.958630+0000 mon.a (mon.0) 32 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.959317+0000 mon.a (mon.0) 33 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.959874+0000 mon.a (mon.0) 34 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.960375+0000 mon.a (mon.0) 35 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.960854+0000 mon.a (mon.0) 36 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.961350+0000 mon.a (mon.0) 37 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:37:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.961838+0000 mon.a (mon.0) 38 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: cluster 2024-04-13T07:37:56.933802+0000 mon.a (mon.0) 20 : cluster 1 Active manager daemon y restarted 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: cluster 2024-04-13T07:37:56.934332+0000 mon.a (mon.0) 21 : cluster 4 Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: cluster 2024-04-13T07:37:56.934682+0000 mon.a (mon.0) 22 : cluster 1 Activating manager daemon y 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: cluster 2024-04-13T07:37:56.941844+0000 mon.a (mon.0) 23 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: cluster 2024-04-13T07:37:56.942365+0000 mon.a (mon.0) 24 : cluster 0 mgrmap e34: y(active, starting, since 0.00795721s), standbys: x 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.951253+0000 mon.a (mon.0) 25 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.951632+0000 mon.a (mon.0) 26 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.952065+0000 mon.a (mon.0) 27 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.956573+0000 mon.a (mon.0) 28 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi089.pccnvq"}]: dispatch 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.957167+0000 mon.a (mon.0) 29 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi203.sjspdy"}]: dispatch 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.957625+0000 mon.a (mon.0) 30 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2024-04-13T07:37:58.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.958001+0000 mon.a (mon.0) 31 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.958630+0000 mon.a (mon.0) 32 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.959317+0000 mon.a (mon.0) 33 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.959874+0000 mon.a (mon.0) 34 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.960375+0000 mon.a (mon.0) 35 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.960854+0000 mon.a (mon.0) 36 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.961350+0000 mon.a (mon.0) 37 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.961838+0000 mon.a (mon.0) 38 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.962351+0000 mon.a (mon.0) 39 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.962862+0000 mon.a (mon.0) 40 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.963429+0000 mon.a (mon.0) 41 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:56.964625+0000 mon.a (mon.0) 42 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: cluster 2024-04-13T07:37:56.973192+0000 mon.a (mon.0) 43 : cluster 1 Manager daemon y is now available 2024-04-13T07:37:58.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:57.015505+0000 mon.a (mon.0) 44 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:57.059085+0000 mon.a (mon.0) 45 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:57.060703+0000 mon.a (mon.0) 46 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:57.063835+0000 mon.a (mon.0) 47 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[31522]: audit 2024-04-13T07:37:57.090481+0000 mon.a (mon.0) 48 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.962351+0000 mon.a (mon.0) 39 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.962862+0000 mon.a (mon.0) 40 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.963429+0000 mon.a (mon.0) 41 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:56.964625+0000 mon.a (mon.0) 42 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: cluster 2024-04-13T07:37:56.973192+0000 mon.a (mon.0) 43 : cluster [INF] Manager daemon y is now available 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:57.015505+0000 mon.a (mon.0) 44 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:37:58.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:57.059085+0000 mon.a (mon.0) 45 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:37:58.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:57.060703+0000 mon.a (mon.0) 46 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2024-04-13T07:37:58.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:57.063835+0000 mon.a (mon.0) 47 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:37:58.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:57 smithi089 bash[82331]: audit 2024-04-13T07:37:57.090481+0000 mon.a (mon.0) 48 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2024-04-13T07:37:58.736 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch upgrade pause' 2024-04-13T07:37:59.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[82331]: cephadm 2024-04-13T07:37:57.834901+0000 mgr.y (mgr.34107) 1 : cephadm [INF] [13/Apr/2024:07:37:57] ENGINE Bus STARTING 2024-04-13T07:37:59.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[82331]: cephadm 2024-04-13T07:37:57.952659+0000 mgr.y (mgr.34107) 2 : cephadm [INF] [13/Apr/2024:07:37:57] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:37:59.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[82331]: audit 2024-04-13T07:37:57.978828+0000 mgr.y (mgr.34107) 3 : audit [DBG] from='client.24793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:37:59.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[82331]: cluster 2024-04-13T07:37:57.980562+0000 mon.a (mon.0) 49 : cluster [DBG] mgrmap e35: y(active, since 1.04613s), standbys: x 2024-04-13T07:37:59.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[82331]: cluster 2024-04-13T07:37:57.986335+0000 mgr.y (mgr.34107) 4 : cluster [DBG] pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:59.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[82331]: cephadm 2024-04-13T07:37:58.054213+0000 mgr.y (mgr.34107) 5 : cephadm [INF] [13/Apr/2024:07:37:58] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:37:59.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[82331]: cephadm 2024-04-13T07:37:58.054368+0000 mgr.y (mgr.34107) 6 : cephadm [INF] [13/Apr/2024:07:37:58] ENGINE Bus STARTED 2024-04-13T07:37:59.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[31522]: cephadm 2024-04-13T07:37:57.834901+0000 mgr.y (mgr.34107) 1 : cephadm 1 [13/Apr/2024:07:37:57] ENGINE Bus STARTING 2024-04-13T07:37:59.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[31522]: cephadm 2024-04-13T07:37:57.952659+0000 mgr.y (mgr.34107) 2 : cephadm 1 [13/Apr/2024:07:37:57] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:37:59.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[31522]: audit 2024-04-13T07:37:57.978828+0000 mgr.y (mgr.34107) 3 : audit 0 from='client.24793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:37:59.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[31522]: cluster 2024-04-13T07:37:57.980562+0000 mon.a (mon.0) 49 : cluster 0 mgrmap e35: y(active, since 1.04613s), standbys: x 2024-04-13T07:37:59.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[31522]: cluster 2024-04-13T07:37:57.986335+0000 mgr.y (mgr.34107) 4 : cluster 0 pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:59.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[31522]: cephadm 2024-04-13T07:37:58.054213+0000 mgr.y (mgr.34107) 5 : cephadm 1 [13/Apr/2024:07:37:58] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:37:59.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:58 smithi089 bash[31522]: cephadm 2024-04-13T07:37:58.054368+0000 mgr.y (mgr.34107) 6 : cephadm 1 [13/Apr/2024:07:37:58] ENGINE Bus STARTED 2024-04-13T07:37:59.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:58 smithi203 bash[23480]: cephadm 2024-04-13T07:37:57.834901+0000 mgr.y (mgr.34107) 1 : cephadm 1 [13/Apr/2024:07:37:57] ENGINE Bus STARTING 2024-04-13T07:37:59.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:58 smithi203 bash[23480]: cephadm 2024-04-13T07:37:57.952659+0000 mgr.y (mgr.34107) 2 : cephadm 1 [13/Apr/2024:07:37:57] ENGINE Serving on https://172.21.15.89:7150 2024-04-13T07:37:59.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:58 smithi203 bash[23480]: audit 2024-04-13T07:37:57.978828+0000 mgr.y (mgr.34107) 3 : audit 0 from='client.24793 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:37:59.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:58 smithi203 bash[23480]: cluster 2024-04-13T07:37:57.980562+0000 mon.a (mon.0) 49 : cluster 0 mgrmap e35: y(active, since 1.04613s), standbys: x 2024-04-13T07:37:59.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:58 smithi203 bash[23480]: cluster 2024-04-13T07:37:57.986335+0000 mgr.y (mgr.34107) 4 : cluster 0 pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:37:59.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:58 smithi203 bash[23480]: cephadm 2024-04-13T07:37:58.054213+0000 mgr.y (mgr.34107) 5 : cephadm 1 [13/Apr/2024:07:37:58] ENGINE Serving on http://172.21.15.89:8765 2024-04-13T07:37:59.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:58 smithi203 bash[23480]: cephadm 2024-04-13T07:37:58.054368+0000 mgr.y (mgr.34107) 6 : cephadm 1 [13/Apr/2024:07:37:58] ENGINE Bus STARTED 2024-04-13T07:38:00.262 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:59 smithi203 bash[23480]: cluster 2024-04-13T07:37:58.958912+0000 mgr.y (mgr.34107) 7 : cluster 0 pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:00.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:37:59 smithi203 bash[23480]: cluster 2024-04-13T07:37:58.984853+0000 mon.a (mon.0) 50 : cluster 0 mgrmap e36: y(active, since 2s), standbys: x 2024-04-13T07:38:00.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:59 smithi089 bash[31522]: cluster 2024-04-13T07:37:58.958912+0000 mgr.y (mgr.34107) 7 : cluster 0 pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:00.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:37:59 smithi089 bash[31522]: cluster 2024-04-13T07:37:58.984853+0000 mon.a (mon.0) 50 : cluster 0 mgrmap e36: y(active, since 2s), standbys: x 2024-04-13T07:38:00.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:59 smithi089 bash[82331]: cluster 2024-04-13T07:37:58.958912+0000 mgr.y (mgr.34107) 7 : cluster [DBG] pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:00.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:37:59 smithi089 bash[82331]: cluster 2024-04-13T07:37:58.984853+0000 mon.a (mon.0) 50 : cluster [DBG] mgrmap e36: y(active, since 2s), standbys: x 2024-04-13T07:38:02.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:01 smithi203 bash[23480]: cluster 2024-04-13T07:38:00.959656+0000 mgr.y (mgr.34107) 8 : cluster 0 pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:02.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:01 smithi203 bash[23480]: cluster 2024-04-13T07:38:00.999884+0000 mon.a (mon.0) 51 : cluster 0 mgrmap e37: y(active, since 4s), standbys: x 2024-04-13T07:38:02.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:01 smithi089 bash[31522]: cluster 2024-04-13T07:38:00.959656+0000 mgr.y (mgr.34107) 8 : cluster 0 pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:02.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:01 smithi089 bash[31522]: cluster 2024-04-13T07:38:00.999884+0000 mon.a (mon.0) 51 : cluster 0 mgrmap e37: y(active, since 4s), standbys: x 2024-04-13T07:38:02.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:01 smithi089 bash[82331]: cluster 2024-04-13T07:38:00.959656+0000 mgr.y (mgr.34107) 8 : cluster [DBG] pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:02.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:01 smithi089 bash[82331]: cluster 2024-04-13T07:38:00.999884+0000 mon.a (mon.0) 51 : cluster [DBG] mgrmap e37: y(active, since 4s), standbys: x 2024-04-13T07:38:03.297 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:38:03.762 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:03 smithi203 bash[60252]: ts=2024-04-13T07:38:03.507Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:04.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[31522]: cluster 2024-04-13T07:38:02.960150+0000 mgr.y (mgr.34107) 9 : cluster 0 pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[31522]: audit 2024-04-13T07:38:03.040201+0000 mon.a (mon.0) 52 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[31522]: audit 2024-04-13T07:38:03.044874+0000 mon.a (mon.0) 53 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[31522]: audit 2024-04-13T07:38:03.550663+0000 mon.a (mon.0) 54 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[31522]: audit 2024-04-13T07:38:03.556105+0000 mon.a (mon.0) 55 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[82331]: cluster 2024-04-13T07:38:02.960150+0000 mgr.y (mgr.34107) 9 : cluster [DBG] pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[82331]: audit 2024-04-13T07:38:03.040201+0000 mon.a (mon.0) 52 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[82331]: audit 2024-04-13T07:38:03.044874+0000 mon.a (mon.0) 53 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[82331]: audit 2024-04-13T07:38:03.550663+0000 mon.a (mon.0) 54 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:04 smithi089 bash[82331]: audit 2024-04-13T07:38:03.556105+0000 mon.a (mon.0) 55 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:04 smithi203 bash[23480]: cluster 2024-04-13T07:38:02.960150+0000 mgr.y (mgr.34107) 9 : cluster 0 pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:04.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:04 smithi203 bash[23480]: audit 2024-04-13T07:38:03.040201+0000 mon.a (mon.0) 52 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:04 smithi203 bash[23480]: audit 2024-04-13T07:38:03.044874+0000 mon.a (mon.0) 53 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:04 smithi203 bash[23480]: audit 2024-04-13T07:38:03.550663+0000 mon.a (mon.0) 54 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:04.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:04 smithi203 bash[23480]: audit 2024-04-13T07:38:03.556105+0000 mon.a (mon.0) 55 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.503 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[31522]: audit 2024-04-13T07:38:04.091746+0000 mon.a (mon.0) 56 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.504 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[31522]: audit 2024-04-13T07:38:04.103386+0000 mon.a (mon.0) 57 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.504 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[31522]: audit 2024-04-13T07:38:04.621584+0000 mon.a (mon.0) 58 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.504 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[31522]: audit 2024-04-13T07:38:04.635447+0000 mon.a (mon.0) 59 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[82331]: audit 2024-04-13T07:38:04.091746+0000 mon.a (mon.0) 56 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[82331]: audit 2024-04-13T07:38:04.103386+0000 mon.a (mon.0) 57 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[82331]: audit 2024-04-13T07:38:04.621584+0000 mon.a (mon.0) 58 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.504 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[82331]: audit 2024-04-13T07:38:04.635447+0000 mon.a (mon.0) 59 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.512 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:05 smithi203 bash[23480]: audit 2024-04-13T07:38:04.091746+0000 mon.a (mon.0) 56 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.512 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:05 smithi203 bash[23480]: audit 2024-04-13T07:38:04.103386+0000 mon.a (mon.0) 57 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:05 smithi203 bash[23480]: audit 2024-04-13T07:38:04.621584+0000 mon.a (mon.0) 58 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:05 smithi203 bash[23480]: audit 2024-04-13T07:38:04.635447+0000 mon.a (mon.0) 59 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:05.938 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:38:06.135 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:38:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:38:05] "GET /metrics HTTP/1.1" 200 31201 "" "Prometheus/2.43.0" 2024-04-13T07:38:06.594 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'sleep 60' 2024-04-13T07:38:07.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:06 smithi203 bash[23480]: cluster 2024-04-13T07:38:04.961287+0000 mgr.y (mgr.34107) 10 : cluster 0 pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:38:07.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:06 smithi203 bash[23480]: audit 2024-04-13T07:38:05.944194+0000 mon.a (mon.0) 60 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:07.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:06 smithi089 bash[82331]: cluster 2024-04-13T07:38:04.961287+0000 mgr.y (mgr.34107) 10 : cluster [DBG] pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:38:07.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:06 smithi089 bash[82331]: audit 2024-04-13T07:38:05.944194+0000 mon.a (mon.0) 60 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:07.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:06 smithi089 bash[31522]: cluster 2024-04-13T07:38:04.961287+0000 mgr.y (mgr.34107) 10 : cluster 0 pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 30 KiB/s rd, 0 B/s wr, 16 op/s 2024-04-13T07:38:07.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:06 smithi089 bash[31522]: audit 2024-04-13T07:38:05.944194+0000 mon.a (mon.0) 60 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:08.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:07 smithi203 bash[23480]: audit 2024-04-13T07:38:05.936634+0000 mgr.y (mgr.34107) 11 : audit 0 from='client.25919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:38:08.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:07 smithi203 bash[23480]: cephadm 2024-04-13T07:38:05.937325+0000 mgr.y (mgr.34107) 12 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:38:08.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:07 smithi089 bash[31522]: audit 2024-04-13T07:38:05.936634+0000 mgr.y (mgr.34107) 11 : audit 0 from='client.25919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:38:08.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:07 smithi089 bash[31522]: cephadm 2024-04-13T07:38:05.937325+0000 mgr.y (mgr.34107) 12 : cephadm 1 Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:38:08.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:07 smithi089 bash[82331]: audit 2024-04-13T07:38:05.936634+0000 mgr.y (mgr.34107) 11 : audit [DBG] from='client.25919 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:38:08.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:07 smithi089 bash[82331]: cephadm 2024-04-13T07:38:05.937325+0000 mgr.y (mgr.34107) 12 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:38:09.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:08 smithi203 bash[23480]: cluster 2024-04-13T07:38:06.961979+0000 mgr.y (mgr.34107) 13 : cluster 0 pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:38:09.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:08 smithi089 bash[82331]: cluster 2024-04-13T07:38:06.961979+0000 mgr.y (mgr.34107) 13 : cluster [DBG] pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:38:09.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:08 smithi089 bash[31522]: cluster 2024-04-13T07:38:06.961979+0000 mgr.y (mgr.34107) 13 : cluster 0 pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 23 KiB/s rd, 0 B/s wr, 12 op/s 2024-04-13T07:38:11.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:10 smithi203 bash[23480]: cluster 2024-04-13T07:38:08.963142+0000 mgr.y (mgr.34107) 14 : cluster 0 pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:38:11.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:10 smithi089 bash[31522]: cluster 2024-04-13T07:38:08.963142+0000 mgr.y (mgr.34107) 14 : cluster 0 pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:38:11.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:10 smithi089 bash[82331]: cluster 2024-04-13T07:38:08.963142+0000 mgr.y (mgr.34107) 14 : cluster [DBG] pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 0 B/s wr, 10 op/s 2024-04-13T07:38:11.336 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:38:12.649 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:12 smithi203 bash[60252]: ts=2024-04-13T07:38:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:12.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:12 smithi089 bash[82331]: cluster 2024-04-13T07:38:10.963771+0000 mgr.y (mgr.34107) 15 : cluster [DBG] pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:12.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:12 smithi089 bash[82331]: audit 2024-04-13T07:38:12.053092+0000 mon.a (mon.0) 61 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:12.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:12 smithi089 bash[31522]: cluster 2024-04-13T07:38:10.963771+0000 mgr.y (mgr.34107) 15 : cluster 0 pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:12.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:12 smithi089 bash[31522]: audit 2024-04-13T07:38:12.053092+0000 mon.a (mon.0) 61 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:13.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:12 smithi203 bash[23480]: cluster 2024-04-13T07:38:10.963771+0000 mgr.y (mgr.34107) 15 : cluster 0 pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:13.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:12 smithi203 bash[23480]: audit 2024-04-13T07:38:12.053092+0000 mon.a (mon.0) 61 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:13.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:13 smithi203 bash[60252]: ts=2024-04-13T07:38:13.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:15.012 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:14 smithi203 bash[23480]: cluster 2024-04-13T07:38:12.964396+0000 mgr.y (mgr.34107) 16 : cluster 0 pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:15.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:14 smithi203 bash[23480]: audit 2024-04-13T07:38:14.491048+0000 mon.a (mon.0) 62 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:15.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:14 smithi203 bash[23480]: audit 2024-04-13T07:38:14.502683+0000 mon.a (mon.0) 63 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:15.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:14 smithi203 bash[23480]: audit 2024-04-13T07:38:14.506187+0000 mon.a (mon.0) 64 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:38:15.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:14 smithi203 bash[23480]: audit 2024-04-13T07:38:14.507955+0000 mon.a (mon.0) 65 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:38:15.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[82331]: cluster 2024-04-13T07:38:12.964396+0000 mgr.y (mgr.34107) 16 : cluster [DBG] pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:15.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[82331]: audit 2024-04-13T07:38:14.491048+0000 mon.a (mon.0) 62 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:15.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[82331]: audit 2024-04-13T07:38:14.502683+0000 mon.a (mon.0) 63 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:15.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[82331]: audit 2024-04-13T07:38:14.506187+0000 mon.a (mon.0) 64 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:38:15.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[82331]: audit 2024-04-13T07:38:14.507955+0000 mon.a (mon.0) 65 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:38:15.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[31522]: cluster 2024-04-13T07:38:12.964396+0000 mgr.y (mgr.34107) 16 : cluster 0 pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:15.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[31522]: audit 2024-04-13T07:38:14.491048+0000 mon.a (mon.0) 62 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:15.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[31522]: audit 2024-04-13T07:38:14.502683+0000 mon.a (mon.0) 63 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:15.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[31522]: audit 2024-04-13T07:38:14.506187+0000 mon.a (mon.0) 64 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:38:15.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:14 smithi089 bash[31522]: audit 2024-04-13T07:38:14.507955+0000 mon.a (mon.0) 65 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:38:16.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:15 smithi203 bash[23480]: cephadm 2024-04-13T07:38:14.510713+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:38:16.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:15 smithi203 bash[23480]: cephadm 2024-04-13T07:38:14.511214+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:38:16.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:15 smithi203 bash[23480]: audit 2024-04-13T07:38:15.281567+0000 mon.a (mon.0) 66 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:15 smithi203 bash[23480]: audit 2024-04-13T07:38:15.288559+0000 mon.a (mon.0) 67 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:15 smithi203 bash[23480]: audit 2024-04-13T07:38:15.298590+0000 mon.a (mon.0) 68 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:15 smithi203 bash[23480]: audit 2024-04-13T07:38:15.305921+0000 mon.a (mon.0) 69 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:15 smithi203 bash[23480]: audit 2024-04-13T07:38:15.312211+0000 mon.a (mon.0) 70 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:15 smithi203 bash[23480]: audit 2024-04-13T07:38:15.390964+0000 mon.a (mon.0) 71 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:38:16.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:38:15] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.43.0" 2024-04-13T07:38:16.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[31522]: cephadm 2024-04-13T07:38:14.510713+0000 mgr.y (mgr.34107) 17 : cephadm 1 Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[31522]: cephadm 2024-04-13T07:38:14.511214+0000 mgr.y (mgr.34107) 18 : cephadm 1 Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[31522]: audit 2024-04-13T07:38:15.281567+0000 mon.a (mon.0) 66 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[31522]: audit 2024-04-13T07:38:15.288559+0000 mon.a (mon.0) 67 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[31522]: audit 2024-04-13T07:38:15.298590+0000 mon.a (mon.0) 68 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[31522]: audit 2024-04-13T07:38:15.305921+0000 mon.a (mon.0) 69 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[31522]: audit 2024-04-13T07:38:15.312211+0000 mon.a (mon.0) 70 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[31522]: audit 2024-04-13T07:38:15.390964+0000 mon.a (mon.0) 71 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[82331]: cephadm 2024-04-13T07:38:14.510713+0000 mgr.y (mgr.34107) 17 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.conf 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[82331]: cephadm 2024-04-13T07:38:14.511214+0000 mgr.y (mgr.34107) 18 : cephadm [INF] Updating smithi203:/etc/ceph/ceph.conf 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[82331]: audit 2024-04-13T07:38:15.281567+0000 mon.a (mon.0) 66 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[82331]: audit 2024-04-13T07:38:15.288559+0000 mon.a (mon.0) 67 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[82331]: audit 2024-04-13T07:38:15.298590+0000 mon.a (mon.0) 68 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[82331]: audit 2024-04-13T07:38:15.305921+0000 mon.a (mon.0) 69 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[82331]: audit 2024-04-13T07:38:15.312211+0000 mon.a (mon.0) 70 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:16.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:15 smithi089 bash[82331]: audit 2024-04-13T07:38:15.390964+0000 mon.a (mon.0) 71 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:38:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: cephadm 2024-04-13T07:38:14.777142+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:38:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: cephadm 2024-04-13T07:38:14.777401+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:38:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: cephadm 2024-04-13T07:38:14.942493+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:38:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: cephadm 2024-04-13T07:38:14.942706+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:38:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: cluster 2024-04-13T07:38:14.965066+0000 mgr.y (mgr.34107) 23 : cluster 0 pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: cephadm 2024-04-13T07:38:15.107777+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:38:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: cephadm 2024-04-13T07:38:15.108095+0000 mgr.y (mgr.34107) 25 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:38:17.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: audit 2024-04-13T07:38:15.689090+0000 mon.a (mon.0) 72 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:38:17.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: audit 2024-04-13T07:38:15.690382+0000 mon.a (mon.0) 73 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:38:17.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:16 smithi203 bash[23480]: audit 2024-04-13T07:38:15.697283+0000 mon.a (mon.0) 74 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: cephadm 2024-04-13T07:38:14.777142+0000 mgr.y (mgr.34107) 19 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: cephadm 2024-04-13T07:38:14.777401+0000 mgr.y (mgr.34107) 20 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: cephadm 2024-04-13T07:38:14.942493+0000 mgr.y (mgr.34107) 21 : cephadm 1 Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: cephadm 2024-04-13T07:38:14.942706+0000 mgr.y (mgr.34107) 22 : cephadm 1 Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: cluster 2024-04-13T07:38:14.965066+0000 mgr.y (mgr.34107) 23 : cluster 0 pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: cephadm 2024-04-13T07:38:15.107777+0000 mgr.y (mgr.34107) 24 : cephadm 1 Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: cephadm 2024-04-13T07:38:15.108095+0000 mgr.y (mgr.34107) 25 : cephadm 1 Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: audit 2024-04-13T07:38:15.689090+0000 mon.a (mon.0) 72 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:38:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: audit 2024-04-13T07:38:15.690382+0000 mon.a (mon.0) 73 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[31522]: audit 2024-04-13T07:38:15.697283+0000 mon.a (mon.0) 74 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: cephadm 2024-04-13T07:38:14.777142+0000 mgr.y (mgr.34107) 19 : cephadm [INF] Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: cephadm 2024-04-13T07:38:14.777401+0000 mgr.y (mgr.34107) 20 : cephadm [INF] Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.conf 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: cephadm 2024-04-13T07:38:14.942493+0000 mgr.y (mgr.34107) 21 : cephadm [INF] Updating smithi089:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: cephadm 2024-04-13T07:38:14.942706+0000 mgr.y (mgr.34107) 22 : cephadm [INF] Updating smithi203:/etc/ceph/ceph.client.admin.keyring 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: cluster 2024-04-13T07:38:14.965066+0000 mgr.y (mgr.34107) 23 : cluster [DBG] pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 0 B/s wr, 9 op/s 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: cephadm 2024-04-13T07:38:15.107777+0000 mgr.y (mgr.34107) 24 : cephadm [INF] Updating smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: cephadm 2024-04-13T07:38:15.108095+0000 mgr.y (mgr.34107) 25 : cephadm [INF] Updating smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/config/ceph.client.admin.keyring 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: audit 2024-04-13T07:38:15.689090+0000 mon.a (mon.0) 72 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: audit 2024-04-13T07:38:15.690382+0000 mon.a (mon.0) 73 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:38:17.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:16 smithi089 bash[82331]: audit 2024-04-13T07:38:15.697283+0000 mon.a (mon.0) 74 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:38:19.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:18 smithi203 bash[23480]: cluster 2024-04-13T07:38:16.965758+0000 mgr.y (mgr.34107) 26 : cluster 0 pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:19.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:18 smithi089 bash[82331]: cluster 2024-04-13T07:38:16.965758+0000 mgr.y (mgr.34107) 26 : cluster [DBG] pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:19.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:18 smithi089 bash[31522]: cluster 2024-04-13T07:38:16.965758+0000 mgr.y (mgr.34107) 26 : cluster 0 pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:21.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:20 smithi203 bash[23480]: cluster 2024-04-13T07:38:18.966925+0000 mgr.y (mgr.34107) 27 : cluster 0 pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:21.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:20 smithi089 bash[31522]: cluster 2024-04-13T07:38:18.966925+0000 mgr.y (mgr.34107) 27 : cluster 0 pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:21.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:20 smithi089 bash[82331]: cluster 2024-04-13T07:38:18.966925+0000 mgr.y (mgr.34107) 27 : cluster [DBG] pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:22.675 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:22 smithi203 bash[60252]: ts=2024-04-13T07:38:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:23.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:22 smithi203 bash[23480]: cluster 2024-04-13T07:38:20.967577+0000 mgr.y (mgr.34107) 28 : cluster 0 pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:22 smithi089 bash[82331]: cluster 2024-04-13T07:38:20.967577+0000 mgr.y (mgr.34107) 28 : cluster [DBG] pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:23.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:22 smithi089 bash[31522]: cluster 2024-04-13T07:38:20.967577+0000 mgr.y (mgr.34107) 28 : cluster 0 pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:23.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:23 smithi203 bash[60252]: ts=2024-04-13T07:38:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:25.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:24 smithi203 bash[23480]: cluster 2024-04-13T07:38:22.968185+0000 mgr.y (mgr.34107) 29 : cluster 0 pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:25.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:24 smithi089 bash[31522]: cluster 2024-04-13T07:38:22.968185+0000 mgr.y (mgr.34107) 29 : cluster 0 pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:25.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:24 smithi089 bash[82331]: cluster 2024-04-13T07:38:22.968185+0000 mgr.y (mgr.34107) 29 : cluster [DBG] pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:26.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:38:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:38:25] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.43.0" 2024-04-13T07:38:27.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:26 smithi203 bash[23480]: cluster 2024-04-13T07:38:24.969282+0000 mgr.y (mgr.34107) 30 : cluster 0 pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:27.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:26 smithi089 bash[31522]: cluster 2024-04-13T07:38:24.969282+0000 mgr.y (mgr.34107) 30 : cluster 0 pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:27.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:26 smithi089 bash[82331]: cluster 2024-04-13T07:38:24.969282+0000 mgr.y (mgr.34107) 30 : cluster [DBG] pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:28.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:27 smithi203 bash[23480]: audit 2024-04-13T07:38:27.053314+0000 mon.a (mon.0) 75 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:28.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:27 smithi089 bash[82331]: audit 2024-04-13T07:38:27.053314+0000 mon.a (mon.0) 75 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:28.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:27 smithi089 bash[31522]: audit 2024-04-13T07:38:27.053314+0000 mon.a (mon.0) 75 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:29.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:28 smithi203 bash[23480]: cluster 2024-04-13T07:38:26.969963+0000 mgr.y (mgr.34107) 31 : cluster 0 pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:29.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:28 smithi089 bash[82331]: cluster 2024-04-13T07:38:26.969963+0000 mgr.y (mgr.34107) 31 : cluster [DBG] pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:29.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:28 smithi089 bash[31522]: cluster 2024-04-13T07:38:26.969963+0000 mgr.y (mgr.34107) 31 : cluster 0 pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:31.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:30 smithi203 bash[23480]: cluster 2024-04-13T07:38:28.971105+0000 mgr.y (mgr.34107) 32 : cluster 0 pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:31.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:30 smithi089 bash[31522]: cluster 2024-04-13T07:38:28.971105+0000 mgr.y (mgr.34107) 32 : cluster 0 pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:31.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:30 smithi089 bash[82331]: cluster 2024-04-13T07:38:28.971105+0000 mgr.y (mgr.34107) 32 : cluster [DBG] pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:32.703 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:32 smithi203 bash[60252]: ts=2024-04-13T07:38:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:33.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:32 smithi203 bash[23480]: cluster 2024-04-13T07:38:30.971725+0000 mgr.y (mgr.34107) 33 : cluster 0 pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:33.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:32 smithi089 bash[82331]: cluster 2024-04-13T07:38:30.971725+0000 mgr.y (mgr.34107) 33 : cluster [DBG] pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:33.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:32 smithi089 bash[31522]: cluster 2024-04-13T07:38:30.971725+0000 mgr.y (mgr.34107) 33 : cluster 0 pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:33.758 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:33 smithi203 bash[60252]: ts=2024-04-13T07:38:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:34.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:33 smithi203 bash[23480]: cluster 2024-04-13T07:38:32.972351+0000 mgr.y (mgr.34107) 34 : cluster 0 pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:34.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:33 smithi089 bash[31522]: cluster 2024-04-13T07:38:32.972351+0000 mgr.y (mgr.34107) 34 : cluster 0 pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:34.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:33 smithi089 bash[82331]: cluster 2024-04-13T07:38:32.972351+0000 mgr.y (mgr.34107) 34 : cluster [DBG] pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:36.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:38:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:38:35] "GET /metrics HTTP/1.1" 200 33890 "" "Prometheus/2.43.0" 2024-04-13T07:38:36.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:36 smithi089 bash[31522]: cluster 2024-04-13T07:38:34.973487+0000 mgr.y (mgr.34107) 35 : cluster 0 pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:36.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:36 smithi089 bash[82331]: cluster 2024-04-13T07:38:34.973487+0000 mgr.y (mgr.34107) 35 : cluster [DBG] pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:36.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:36 smithi203 bash[23480]: cluster 2024-04-13T07:38:34.973487+0000 mgr.y (mgr.34107) 35 : cluster 0 pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:38.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:38 smithi089 bash[82331]: cluster 2024-04-13T07:38:36.974152+0000 mgr.y (mgr.34107) 36 : cluster [DBG] pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:38.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:38 smithi089 bash[31522]: cluster 2024-04-13T07:38:36.974152+0000 mgr.y (mgr.34107) 36 : cluster 0 pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:38.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:38 smithi203 bash[23480]: cluster 2024-04-13T07:38:36.974152+0000 mgr.y (mgr.34107) 36 : cluster 0 pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:40.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:40 smithi089 bash[82331]: cluster 2024-04-13T07:38:38.975277+0000 mgr.y (mgr.34107) 37 : cluster [DBG] pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:40.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:40 smithi089 bash[31522]: cluster 2024-04-13T07:38:38.975277+0000 mgr.y (mgr.34107) 37 : cluster 0 pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:40.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:40 smithi203 bash[23480]: cluster 2024-04-13T07:38:38.975277+0000 mgr.y (mgr.34107) 37 : cluster 0 pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:42.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:42 smithi089 bash[31522]: cluster 2024-04-13T07:38:40.975881+0000 mgr.y (mgr.34107) 38 : cluster 0 pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:42 smithi089 bash[82331]: cluster 2024-04-13T07:38:40.975881+0000 mgr.y (mgr.34107) 38 : cluster [DBG] pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:42.307 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:42 smithi203 bash[23480]: cluster 2024-04-13T07:38:40.975881+0000 mgr.y (mgr.34107) 38 : cluster 0 pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:42.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:42 smithi203 bash[60252]: ts=2024-04-13T07:38:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:43.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:43 smithi089 bash[31522]: audit 2024-04-13T07:38:42.054482+0000 mon.a (mon.0) 76 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:43.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:43 smithi089 bash[82331]: audit 2024-04-13T07:38:42.054482+0000 mon.a (mon.0) 76 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:43.501 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:43 smithi203 bash[23480]: audit 2024-04-13T07:38:42.054482+0000 mon.a (mon.0) 76 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:43.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:43 smithi203 bash[60252]: ts=2024-04-13T07:38:43.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:44 smithi089 bash[82331]: cluster 2024-04-13T07:38:42.976539+0000 mgr.y (mgr.34107) 39 : cluster [DBG] pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:44 smithi089 bash[31522]: cluster 2024-04-13T07:38:42.976539+0000 mgr.y (mgr.34107) 39 : cluster 0 pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:44.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:44 smithi203 bash[23480]: cluster 2024-04-13T07:38:42.976539+0000 mgr.y (mgr.34107) 39 : cluster 0 pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:46.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:46 smithi089 bash[82331]: cluster 2024-04-13T07:38:44.977575+0000 mgr.y (mgr.34107) 40 : cluster [DBG] pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:46.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:38:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:38:45] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.43.0" 2024-04-13T07:38:46.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:46 smithi089 bash[31522]: cluster 2024-04-13T07:38:44.977575+0000 mgr.y (mgr.34107) 40 : cluster 0 pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:46.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:46 smithi203 bash[23480]: cluster 2024-04-13T07:38:44.977575+0000 mgr.y (mgr.34107) 40 : cluster 0 pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:48.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:48 smithi203 bash[23480]: cluster 2024-04-13T07:38:46.978224+0000 mgr.y (mgr.34107) 41 : cluster 0 pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:48.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:48 smithi089 bash[82331]: cluster 2024-04-13T07:38:46.978224+0000 mgr.y (mgr.34107) 41 : cluster [DBG] pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:48.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:48 smithi089 bash[31522]: cluster 2024-04-13T07:38:46.978224+0000 mgr.y (mgr.34107) 41 : cluster 0 pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:50.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:50 smithi203 bash[23480]: cluster 2024-04-13T07:38:48.979322+0000 mgr.y (mgr.34107) 42 : cluster 0 pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:50.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:50 smithi089 bash[31522]: cluster 2024-04-13T07:38:48.979322+0000 mgr.y (mgr.34107) 42 : cluster 0 pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:50.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:50 smithi089 bash[82331]: cluster 2024-04-13T07:38:48.979322+0000 mgr.y (mgr.34107) 42 : cluster [DBG] pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:52.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:52 smithi203 bash[23480]: cluster 2024-04-13T07:38:50.979923+0000 mgr.y (mgr.34107) 43 : cluster 0 pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:52.514 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:52 smithi203 bash[60252]: ts=2024-04-13T07:38:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:52.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:52 smithi089 bash[82331]: cluster 2024-04-13T07:38:50.979923+0000 mgr.y (mgr.34107) 43 : cluster [DBG] pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:52.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:52 smithi089 bash[31522]: cluster 2024-04-13T07:38:50.979923+0000 mgr.y (mgr.34107) 43 : cluster 0 pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:53.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:38:53 smithi203 bash[60252]: ts=2024-04-13T07:38:53.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:38:54.513 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:54 smithi203 bash[23480]: cluster 2024-04-13T07:38:52.980619+0000 mgr.y (mgr.34107) 44 : cluster 0 pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:54.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:54 smithi089 bash[31522]: cluster 2024-04-13T07:38:52.980619+0000 mgr.y (mgr.34107) 44 : cluster 0 pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:54.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:54 smithi089 bash[82331]: cluster 2024-04-13T07:38:52.980619+0000 mgr.y (mgr.34107) 44 : cluster [DBG] pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:56.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:38:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:38:55] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.43.0" 2024-04-13T07:38:57.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:56 smithi203 bash[23480]: cluster 2024-04-13T07:38:54.981724+0000 mgr.y (mgr.34107) 45 : cluster 0 pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:56 smithi089 bash[82331]: cluster 2024-04-13T07:38:54.981724+0000 mgr.y (mgr.34107) 45 : cluster [DBG] pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:56 smithi089 bash[31522]: cluster 2024-04-13T07:38:54.981724+0000 mgr.y (mgr.34107) 45 : cluster 0 pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:58.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:57 smithi203 bash[23480]: audit 2024-04-13T07:38:57.059756+0000 mon.a (mon.0) 77 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:58.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:57 smithi089 bash[31522]: audit 2024-04-13T07:38:57.059756+0000 mon.a (mon.0) 77 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:58.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:57 smithi089 bash[82331]: audit 2024-04-13T07:38:57.059756+0000 mon.a (mon.0) 77 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:38:59.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:38:58 smithi203 bash[23480]: cluster 2024-04-13T07:38:56.982367+0000 mgr.y (mgr.34107) 46 : cluster 0 pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:59.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:38:58 smithi089 bash[31522]: cluster 2024-04-13T07:38:56.982367+0000 mgr.y (mgr.34107) 46 : cluster 0 pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:38:59.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:38:58 smithi089 bash[82331]: cluster 2024-04-13T07:38:56.982367+0000 mgr.y (mgr.34107) 46 : cluster [DBG] pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:01.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:00 smithi203 bash[23480]: cluster 2024-04-13T07:38:58.983536+0000 mgr.y (mgr.34107) 47 : cluster 0 pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:01.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:00 smithi089 bash[31522]: cluster 2024-04-13T07:38:58.983536+0000 mgr.y (mgr.34107) 47 : cluster 0 pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:01.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:00 smithi089 bash[82331]: cluster 2024-04-13T07:38:58.983536+0000 mgr.y (mgr.34107) 47 : cluster [DBG] pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:02.582 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:02 smithi203 bash[60252]: ts=2024-04-13T07:39:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:03.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:02 smithi203 bash[23480]: cluster 2024-04-13T07:39:00.984162+0000 mgr.y (mgr.34107) 48 : cluster 0 pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:03.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:02 smithi089 bash[31522]: cluster 2024-04-13T07:39:00.984162+0000 mgr.y (mgr.34107) 48 : cluster 0 pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:03.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:02 smithi089 bash[82331]: cluster 2024-04-13T07:39:00.984162+0000 mgr.y (mgr.34107) 48 : cluster [DBG] pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:03.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:03 smithi203 bash[60252]: ts=2024-04-13T07:39:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:05.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:04 smithi203 bash[23480]: cluster 2024-04-13T07:39:02.984780+0000 mgr.y (mgr.34107) 49 : cluster 0 pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:05.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:04 smithi089 bash[31522]: cluster 2024-04-13T07:39:02.984780+0000 mgr.y (mgr.34107) 49 : cluster 0 pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:05.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:04 smithi089 bash[82331]: cluster 2024-04-13T07:39:02.984780+0000 mgr.y (mgr.34107) 49 : cluster [DBG] pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:06.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:39:05] "GET /metrics HTTP/1.1" 200 33882 "" "Prometheus/2.43.0" 2024-04-13T07:39:07.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:06 smithi203 bash[23480]: cluster 2024-04-13T07:39:04.985867+0000 mgr.y (mgr.34107) 50 : cluster 0 pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:07.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:06 smithi089 bash[31522]: cluster 2024-04-13T07:39:04.985867+0000 mgr.y (mgr.34107) 50 : cluster 0 pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:07.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:06 smithi089 bash[82331]: cluster 2024-04-13T07:39:04.985867+0000 mgr.y (mgr.34107) 50 : cluster [DBG] pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:09.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:08 smithi203 bash[23480]: cluster 2024-04-13T07:39:06.986501+0000 mgr.y (mgr.34107) 51 : cluster 0 pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:09.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:08 smithi089 bash[31522]: cluster 2024-04-13T07:39:06.986501+0000 mgr.y (mgr.34107) 51 : cluster 0 pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:09.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:08 smithi089 bash[82331]: cluster 2024-04-13T07:39:06.986501+0000 mgr.y (mgr.34107) 51 : cluster [DBG] pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:11.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:10 smithi203 bash[23480]: cluster 2024-04-13T07:39:08.987587+0000 mgr.y (mgr.34107) 52 : cluster 0 pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:11.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:10 smithi089 bash[31522]: cluster 2024-04-13T07:39:08.987587+0000 mgr.y (mgr.34107) 52 : cluster 0 pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:11.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:10 smithi089 bash[82331]: cluster 2024-04-13T07:39:08.987587+0000 mgr.y (mgr.34107) 52 : cluster [DBG] pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:12.601 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:12 smithi203 bash[60252]: ts=2024-04-13T07:39:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:13.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:12 smithi203 bash[23480]: cluster 2024-04-13T07:39:10.988235+0000 mgr.y (mgr.34107) 53 : cluster 0 pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:13.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:12 smithi203 bash[23480]: audit 2024-04-13T07:39:12.060190+0000 mon.a (mon.0) 78 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:39:13.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:12 smithi089 bash[31522]: cluster 2024-04-13T07:39:10.988235+0000 mgr.y (mgr.34107) 53 : cluster 0 pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:13.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:12 smithi089 bash[31522]: audit 2024-04-13T07:39:12.060190+0000 mon.a (mon.0) 78 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:39:13.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:12 smithi089 bash[82331]: cluster 2024-04-13T07:39:10.988235+0000 mgr.y (mgr.34107) 53 : cluster [DBG] pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:13.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:12 smithi089 bash[82331]: audit 2024-04-13T07:39:12.060190+0000 mon.a (mon.0) 78 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:39:13.763 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:13 smithi203 bash[60252]: ts=2024-04-13T07:39:13.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:14.370 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch upgrade resume' 2024-04-13T07:39:14.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:14 smithi089 bash[82331]: cluster 2024-04-13T07:39:12.989015+0000 mgr.y (mgr.34107) 54 : cluster [DBG] pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:14.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:14 smithi089 bash[31522]: cluster 2024-04-13T07:39:12.989015+0000 mgr.y (mgr.34107) 54 : cluster 0 pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:15.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:14 smithi203 bash[23480]: cluster 2024-04-13T07:39:12.989015+0000 mgr.y (mgr.34107) 54 : cluster 0 pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:16.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:39:15] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.43.0" 2024-04-13T07:39:17.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:16 smithi203 bash[23480]: cluster 2024-04-13T07:39:14.990057+0000 mgr.y (mgr.34107) 55 : cluster 0 pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:17.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:16 smithi203 bash[23480]: audit 2024-04-13T07:39:15.772168+0000 mon.a (mon.0) 79 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:17.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:16 smithi089 bash[82331]: cluster 2024-04-13T07:39:14.990057+0000 mgr.y (mgr.34107) 55 : cluster [DBG] pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:17.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:16 smithi089 bash[82331]: audit 2024-04-13T07:39:15.772168+0000 mon.a (mon.0) 79 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:16 smithi089 bash[31522]: cluster 2024-04-13T07:39:14.990057+0000 mgr.y (mgr.34107) 55 : cluster 0 pgmap v42: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:16 smithi089 bash[31522]: audit 2024-04-13T07:39:15.772168+0000 mon.a (mon.0) 79 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:19.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:18 smithi203 bash[23480]: cluster 2024-04-13T07:39:16.990638+0000 mgr.y (mgr.34107) 56 : cluster 0 pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:19.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:18 smithi089 bash[82331]: cluster 2024-04-13T07:39:16.990638+0000 mgr.y (mgr.34107) 56 : cluster [DBG] pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:19.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:18 smithi089 bash[31522]: cluster 2024-04-13T07:39:16.990638+0000 mgr.y (mgr.34107) 56 : cluster 0 pgmap v43: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:19.104 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:39:21.013 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:20 smithi203 bash[23480]: cluster 2024-04-13T07:39:18.991740+0000 mgr.y (mgr.34107) 57 : cluster 0 pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:21.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:20 smithi089 bash[82331]: cluster 2024-04-13T07:39:18.991740+0000 mgr.y (mgr.34107) 57 : cluster [DBG] pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:21.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:20 smithi089 bash[31522]: cluster 2024-04-13T07:39:18.991740+0000 mgr.y (mgr.34107) 57 : cluster 0 pgmap v44: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:21.731 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:39:22.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:21 smithi089 bash[82331]: audit 2024-04-13T07:39:20.861675+0000 mon.a (mon.0) 80 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:21 smithi089 bash[82331]: audit 2024-04-13T07:39:20.869032+0000 mon.a (mon.0) 81 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:21 smithi089 bash[82331]: cluster 2024-04-13T07:39:20.992330+0000 mgr.y (mgr.34107) 58 : cluster [DBG] pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:22.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:21 smithi089 bash[82331]: audit 2024-04-13T07:39:21.736973+0000 mon.a (mon.0) 82 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:21 smithi089 bash[31522]: audit 2024-04-13T07:39:20.861675+0000 mon.a (mon.0) 80 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:21 smithi089 bash[31522]: audit 2024-04-13T07:39:20.869032+0000 mon.a (mon.0) 81 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:21 smithi089 bash[31522]: cluster 2024-04-13T07:39:20.992330+0000 mgr.y (mgr.34107) 58 : cluster 0 pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:22.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:21 smithi089 bash[31522]: audit 2024-04-13T07:39:21.736973+0000 mon.a (mon.0) 82 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.263 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:21 smithi203 bash[23480]: audit 2024-04-13T07:39:20.861675+0000 mon.a (mon.0) 80 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:21 smithi203 bash[23480]: audit 2024-04-13T07:39:20.869032+0000 mon.a (mon.0) 81 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:21 smithi203 bash[23480]: cluster 2024-04-13T07:39:20.992330+0000 mgr.y (mgr.34107) 58 : cluster 0 pgmap v45: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:22.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:21 smithi203 bash[23480]: audit 2024-04-13T07:39:21.736973+0000 mon.a (mon.0) 82 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:22.414 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'echo wait for majority of mons to upgrade' 2024-04-13T07:39:22.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[60252]: ts=2024-04-13T07:39:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:23.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:21.731629+0000 mgr.y (mgr.34107) 59 : audit 0 from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:39:23.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: cephadm 2024-04-13T07:39:21.732220+0000 mgr.y (mgr.34107) 60 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:39:23.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:21.876081+0000 mon.a (mon.0) 83 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:21.893262+0000 mon.a (mon.0) 84 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:22.574305+0000 mon.a (mon.0) 85 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:23.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:22.575138+0000 mon.a (mon.0) 86 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:39:23.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:22.579938+0000 mon.a (mon.0) 87 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:22.639121+0000 mon.a (mon.0) 88 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:23.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:22.641494+0000 mon.a (mon.0) 89 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:39:23.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:22.643141+0000 mon.a (mon.0) 90 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:39:23.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: audit 2024-04-13T07:39:22.644099+0000 mon.a (mon.0) 91 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-13T07:39:23.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:22 smithi203 bash[23480]: cephadm 2024-04-13T07:39:22.644732+0000 mgr.y (mgr.34107) 61 : cephadm 1 Upgrade: It appears safe to stop mon.c 2024-04-13T07:39:23.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:21.731629+0000 mgr.y (mgr.34107) 59 : audit [DBG] from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:39:23.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: cephadm 2024-04-13T07:39:21.732220+0000 mgr.y (mgr.34107) 60 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:39:23.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:21.876081+0000 mon.a (mon.0) 83 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:21.893262+0000 mon.a (mon.0) 84 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:22.574305+0000 mon.a (mon.0) 85 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:23.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:22.575138+0000 mon.a (mon.0) 86 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:39:23.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:22.579938+0000 mon.a (mon.0) 87 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:22.639121+0000 mon.a (mon.0) 88 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:22.641494+0000 mon.a (mon.0) 89 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:22.643141+0000 mon.a (mon.0) 90 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: audit 2024-04-13T07:39:22.644099+0000 mon.a (mon.0) 91 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[82331]: cephadm 2024-04-13T07:39:22.644732+0000 mgr.y (mgr.34107) 61 : cephadm [INF] Upgrade: It appears safe to stop mon.c 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:21.731629+0000 mgr.y (mgr.34107) 59 : audit 0 from='client.34167 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: cephadm 2024-04-13T07:39:21.732220+0000 mgr.y (mgr.34107) 60 : cephadm 1 Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:21.876081+0000 mon.a (mon.0) 83 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:21.893262+0000 mon.a (mon.0) 84 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:22.574305+0000 mon.a (mon.0) 85 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:23.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:22.575138+0000 mon.a (mon.0) 86 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:39:23.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:22.579938+0000 mon.a (mon.0) 87 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:23.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:22.639121+0000 mon.a (mon.0) 88 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:23.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:22.641494+0000 mon.a (mon.0) 89 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:39:23.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:22.643141+0000 mon.a (mon.0) 90 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:39:23.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: audit 2024-04-13T07:39:22.644099+0000 mon.a (mon.0) 91 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2024-04-13T07:39:23.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:22 smithi089 bash[31522]: cephadm 2024-04-13T07:39:22.644732+0000 mgr.y (mgr.34107) 61 : cephadm 1 Upgrade: It appears safe to stop mon.c 2024-04-13T07:39:23.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:23 smithi203 bash[60252]: ts=2024-04-13T07:39:23.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:24.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:23 smithi203 bash[23480]: cluster 2024-04-13T07:39:22.992988+0000 mgr.y (mgr.34107) 62 : cluster 0 pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:24.264 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:23 smithi089 bash[31522]: cluster 2024-04-13T07:39:22.992988+0000 mgr.y (mgr.34107) 62 : cluster 0 pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:24.264 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:23 smithi089 bash[82331]: cluster 2024-04-13T07:39:22.992988+0000 mgr.y (mgr.34107) 62 : cluster [DBG] pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:26.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:25 smithi203 bash[23480]: cephadm 2024-04-13T07:39:24.713263+0000 mgr.y (mgr.34107) 63 : cephadm 1 Upgrade: Updating mon.c 2024-04-13T07:39:26.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:25 smithi203 bash[23480]: audit 2024-04-13T07:39:24.719597+0000 mon.a (mon.0) 92 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:26.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:25 smithi203 bash[23480]: audit 2024-04-13T07:39:24.722744+0000 mon.a (mon.0) 93 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:39:26.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:25 smithi203 bash[23480]: audit 2024-04-13T07:39:24.724352+0000 mon.a (mon.0) 94 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:39:26.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:25 smithi203 bash[23480]: audit 2024-04-13T07:39:24.725842+0000 mon.a (mon.0) 95 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:26.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:39:25] "GET /metrics HTTP/1.1" 200 33887 "" "Prometheus/2.43.0" 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[31522]: cephadm 2024-04-13T07:39:24.713263+0000 mgr.y (mgr.34107) 63 : cephadm 1 Upgrade: Updating mon.c 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[31522]: audit 2024-04-13T07:39:24.719597+0000 mon.a (mon.0) 92 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[31522]: audit 2024-04-13T07:39:24.722744+0000 mon.a (mon.0) 93 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[31522]: audit 2024-04-13T07:39:24.724352+0000 mon.a (mon.0) 94 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[31522]: audit 2024-04-13T07:39:24.725842+0000 mon.a (mon.0) 95 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[82331]: cephadm 2024-04-13T07:39:24.713263+0000 mgr.y (mgr.34107) 63 : cephadm [INF] Upgrade: Updating mon.c 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[82331]: audit 2024-04-13T07:39:24.719597+0000 mon.a (mon.0) 92 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[82331]: audit 2024-04-13T07:39:24.722744+0000 mon.a (mon.0) 93 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:39:26.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[82331]: audit 2024-04-13T07:39:24.724352+0000 mon.a (mon.0) 94 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:39:26.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:25 smithi089 bash[82331]: audit 2024-04-13T07:39:24.725842+0000 mon.a (mon.0) 95 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:26.858 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:26.859 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:26.859 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:26.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:26 smithi089 bash[31522]: cephadm 2024-04-13T07:39:24.727631+0000 mgr.y (mgr.34107) 64 : cephadm 1 Deploying daemon mon.c on smithi089 2024-04-13T07:39:26.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:26 smithi089 bash[31522]: cluster 2024-04-13T07:39:24.993865+0000 mgr.y (mgr.34107) 65 : cluster 0 pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:26.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:26.860 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: Stopping Ceph mon.c for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:39:26.860 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:26.860 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:26.861 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:26 smithi089 bash[82331]: cephadm 2024-04-13T07:39:24.727631+0000 mgr.y (mgr.34107) 64 : cephadm [INF] Deploying daemon mon.c on smithi089 2024-04-13T07:39:26.861 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:26 smithi089 bash[82331]: cluster 2024-04-13T07:39:24.993865+0000 mgr.y (mgr.34107) 65 : cluster [DBG] pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:26.861 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:26.861 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:26.862 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:39:26 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:27.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:26 smithi203 bash[23480]: cephadm 2024-04-13T07:39:24.727631+0000 mgr.y (mgr.34107) 64 : cephadm 1 Deploying daemon mon.c on smithi089 2024-04-13T07:39:27.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:26 smithi203 bash[23480]: cluster 2024-04-13T07:39:24.993865+0000 mgr.y (mgr.34107) 65 : cluster 0 pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:27.116 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:26 smithi089 bash[31522]: debug 2024-04-13T07:39:26.966+0000 7fc97c797700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-13T07:39:27.117 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:26 smithi089 bash[31522]: debug 2024-04-13T07:39:26.966+0000 7fc97c797700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2024-04-13T07:39:27.200 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:39:27.973 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:27 smithi089 bash[84625]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mon-c 2024-04-13T07:39:28.253 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.253 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.254 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.254 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:27 smithi089 bash[84695]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mon-c 2024-04-13T07:39:28.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:27 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.c.service: Deactivated successfully. 2024-04-13T07:39:28.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:27 smithi089 systemd[1]: Stopped Ceph mon.c for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:39:28.254 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.255 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.255 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.255 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.256 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:28.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:28 smithi089 systemd[1]: Started Ceph mon.c for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:39:30.044 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.786+0000 7f691eaf6c80 0 set uid:gid to 167:167 (ceph:ceph) 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.786+0000 7f691eaf6c80 0 ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev), process ceph-mon, pid 7 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.786+0000 7f691eaf6c80 0 pidfile_write: ignore empty --pid-file 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 0 load: jerasure load: lrc 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: RocksDB version: 7.9.2 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Git sha 0 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Compile date 2024-04-12 15:38:20 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: DB SUMMARY 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: DB Session ID: UEO4S66C5KAZAK1BLV7N 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: CURRENT file: CURRENT 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: IDENTITY file: IDENTITY 2024-04-13T07:39:30.045 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 2120 Bytes 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000036.sst 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000034.log size: 496754 ; 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.error_if_exists: 0 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.create_if_missing: 0 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.paranoid_checks: 1 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.env: 0x55a258650cc0 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.fs: PosixFileSystem 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.info_log: 0x55a25a77fc40 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_file_opening_threads: 16 2024-04-13T07:39:30.046 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.statistics: (nil) 2024-04-13T07:39:30.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.use_fsync: 0 2024-04-13T07:39:30.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_log_file_size: 0 2024-04-13T07:39:30.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-13T07:39:30.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.keep_log_file_num: 1000 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.recycle_log_file_num: 0 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.allow_fallocate: 1 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.allow_mmap_reads: 0 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.allow_mmap_writes: 0 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.use_direct_reads: 0 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.create_missing_column_families: 0 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.db_log_dir: 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.wal_dir: 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-04-13T07:39:30.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.advise_random_on_open: 1 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.db_write_buffer_size: 0 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.write_buffer_manager: 0x55a25a752820 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.rate_limiter: (nil) 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.wal_recovery_mode: 2 2024-04-13T07:39:30.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.enable_thread_tracking: 0 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.enable_pipelined_write: 0 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.unordered_write: 0 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.row_cache: None 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.wal_filter: None 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.allow_ingest_behind: 0 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.two_write_queues: 0 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.manual_wal_flush: 0 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.wal_compression: 0 2024-04-13T07:39:30.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.atomic_flush: 0 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.log_readahead_size: 0 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.best_efforts_recovery: 0 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.allow_data_in_errors: 0 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.db_host_id: __hostname__ 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_background_jobs: 2 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_background_compactions: -1 2024-04-13T07:39:30.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_subcompactions: 1 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_total_wal_size: 0 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_open_files: -1 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bytes_per_sync: 0 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_readahead_size: 0 2024-04-13T07:39:30.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_background_flushes: -1 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Compression algorithms supported: 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: kZSTD supported: 0 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: kXpressCompression supported: 0 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: kLZ4HCCompression supported: 1 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: kZlibCompression supported: 1 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: kSnappyCompression supported: 1 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: kLZ4Compression supported: 1 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: kBZip2Compression supported: 0 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-04-13T07:39:30.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.merge_operator: 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_filter: None 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_filter_factory: None 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.sst_partitioner_factory: None 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55a25a77fe00) 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: cache_index_and_filter_blocks: 1 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-13T07:39:30.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: pin_top_level_index_and_filter: 1 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: index_type: 0 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: data_block_index_type: 0 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: index_shortening: 1 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: data_block_hash_table_util_ratio: 0.750000 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: checksum: 4 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: no_block_cache: 0 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: block_cache: 0x55a25a783090 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: block_cache_name: BinnedLRUCache 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: block_cache_options: 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: capacity : 536870912 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: num_shard_bits : 4 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: strict_capacity_limit : 0 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: high_pri_pool_ratio: 0.000 2024-04-13T07:39:30.056 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: block_cache_compressed: (nil) 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: persistent_cache: (nil) 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: block_size: 4096 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: block_size_deviation: 10 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: block_restart_interval: 16 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: index_block_restart_interval: 1 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: metadata_block_size: 4096 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: partition_filters: 0 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: use_delta_encoding: 1 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: filter_policy: bloomfilter 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: whole_key_filtering: 1 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: verify_compression: 0 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: read_amp_bytes_per_bit: 0 2024-04-13T07:39:30.057 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: format_version: 5 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: enable_index_compression: 1 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: block_align: 0 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: max_auto_readahead_size: 262144 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: prepopulate_block_cache: 0 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: initial_auto_readahead_size: 8192 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: num_file_reads_for_auto_readahead: 2 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.write_buffer_size: 33554432 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_write_buffer_number: 2 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression: NoCompression 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression: Disabled 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.prefix_extractor: nullptr 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-13T07:39:30.058 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.num_levels: 7 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-04-13T07:39:30.059 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.level: 32767 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.strategy: 0 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.enabled: false 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.target_file_size_base: 67108864 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-13T07:39:30.060 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.arena_block_size: 1048576 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-13T07:39:30.061 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.disable_auto_compactions: 0 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.inplace_update_support: 0 2024-04-13T07:39:30.062 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.bloom_locality: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.max_successive_merges: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.paranoid_file_checks: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.force_consistency_checks: 1 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.report_bg_io_stats: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.ttl: 2592000 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-13T07:39:30.063 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.enable_blob_files: false 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.min_blob_size: 0 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.blob_file_size: 268435456 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.blob_file_starting_level: 0 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 38, last_sequence is 18338, log_number is 34,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 34 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 34 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 81309d95-539e-40bd-9b83-246e0be98e37 2024-04-13T07:39:30.064 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993969794604, "job": 1, "event": "recovery_started", "wal_files": [34]} 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.790+0000 7f691eaf6c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #34 mode 2 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f691eaf6c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993969797176, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 39, "file_size": 326909, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18343, "largest_seqno": 18709, "table_properties": {"data_size": 324426, "index_size": 1059, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 389, "raw_key_size": 3453, "raw_average_key_size": 23, "raw_value_size": 320983, "raw_average_value_size": 2154, "num_data_blocks": 46, "num_entries": 149, "num_filter_entries": 149, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712993969, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "81309d95-539e-40bd-9b83-246e0be98e37", "db_session_id": "UEO4S66C5KAZAK1BLV7N", "orig_file_number": 39, "seqno_to_time_mapping": "N/A"}} 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f691eaf6c80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993969797289, "job": 1, "event": "recovery_finished"} 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f691eaf6c80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 41 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f691eaf6c80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f691eaf6c80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000034.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f691eaf6c80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x55a25a876000 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f691eaf6c80 4 rocksdb: DB pointer 0x55a25a860000 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f690d250700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.794+0000 7f690d250700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: ** DB Stats ** 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:39:30.065 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: ** Compaction Stats [default] ** 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: L0 1/0 319.25 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 210.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: L6 1/0 8.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Sum 2/0 8.66 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 210.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 210.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: ** Compaction Stats [default] ** 2024-04-13T07:39:30.066 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 210.4 0.00 0.00 1 0.001 0 0 0.0 0.0 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Flush(GB): cumulative 0.000, interval 0.000 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: AddFile(Total Files): cumulative 0, interval 0 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: AddFile(L0 Files): cumulative 0, interval 0 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: AddFile(Keys): cumulative 0, interval 0 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Cumulative compaction: 0.00 GB write, 58.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Interval compaction: 0.00 GB write, 58.69 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-13T07:39:30.067 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Block cache BinnedLRUCache@0x55a25a783090#7 capacity: 512.00 MB usage: 1.64 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Block cache entry stats(count,size,portion): FilterBlock(1,0.44 KB,8.34465e-05%) IndexBlock(1,1.20 KB,0.000229478%) Misc(1,0.00 KB,0%) 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: ** File Read Latency Histogram By Level [default] ** 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 0 starting mon.c rank 2 at public addrs [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] at bind addrs [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 1 mon.c@-1(???) e3 preinit fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 0 mon.c@-1(???).mds e10 new map 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 0 mon.c@-1(???).mds e10 print_map 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: e10 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: enable_multiple, ever_enabled_multiple: 1,1 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: legacy client fscid: 1 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Filesystem 'foo' (1) 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: fs_name foo 2024-04-13T07:39:30.068 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: epoch 10 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: created 2024-04-13T07:32:03.054183+0000 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: modified 2024-04-13T07:32:13.856309+0000 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: tableserver 0 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: root 0 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: session_timeout 60 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: session_autoclose 300 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: max_file_size 1099511627776 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: max_xattr_size 65536 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: required_client_features {} 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: last_failure 0 2024-04-13T07:39:30.069 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: last_failure_osd_epoch 70 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: max_mds 1 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: in 0 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: up {0=24364} 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: failed 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: damaged 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: stopped 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: data_pools [4] 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: metadata_pool 3 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: inline_data disabled 2024-04-13T07:39:30.070 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: balancer 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: bal_rank_mask -1 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: standby_count_wanted 1 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: qdb_cluster leader: 0 members: 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: [mds.foo.smithi203.sjspdy{0:24364} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] compat {c=[1],r=[1],i=[7ff]}] 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: Standby daemons: 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: [mds.foo.smithi089.pccnvq{-1:14583} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.89:6834/2537258916,v1:172.21.15.89:6835/2537258916] compat {c=[1],r=[1],i=[7ff]}] 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 0 mon.c@-1(???).osd e92 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 0 mon.c@-1(???).osd e92 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 0 mon.c@-1(???).osd e92 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 0 mon.c@-1(???).osd e92 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:39:30.071 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:29 smithi089 bash[84777]: debug 2024-04-13T07:39:29.798+0000 7f691eaf6c80 1 mon.c@-1(???).paxosservice(auth 1..22) refresh upgraded, format 0 -> 3 2024-04-13T07:39:30.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:30.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:30 smithi089 bash[82331]: debug 2024-04-13T07:39:30.046+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:30.684 INFO:teuthology.orchestra.run.smithi089.stdout:wait for majority of mons to upgrade 2024-04-13T07:39:31.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:28.995478+0000 mgr.y (mgr.34107) 67 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:31.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.013052+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.014768+0000 mon.a (mon.0) 97 : cluster [INF] mon.a calling monitor election 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.015909+0000 mon.b (mon.1) 47 : cluster [INF] mon.b calling monitor election 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.040688+0000 mon.a (mon.0) 98 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047722+0000 mon.a (mon.0) 99 : cluster [DBG] monmap epoch 3 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047751+0000 mon.a (mon.0) 100 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047774+0000 mon.a (mon.0) 101 : cluster [DBG] last_changed 2024-04-13T07:21:29.816921+0000 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047799+0000 mon.a (mon.0) 102 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047844+0000 mon.a (mon.0) 103 : cluster [DBG] min_mon_release 18 (reef) 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047873+0000 mon.a (mon.0) 104 : cluster [DBG] election_strategy: 1 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047891+0000 mon.a (mon.0) 105 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047917+0000 mon.a (mon.0) 106 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.047931+0000 mon.a (mon.0) 107 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:39:31.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.048708+0000 mon.a (mon.0) 108 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.048749+0000 mon.a (mon.0) 109 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049192+0000 mon.a (mon.0) 110 : cluster [DBG] mgrmap e37: y(active, since 93s), standbys: x 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049425+0000 mon.a (mon.0) 111 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049454+0000 mon.a (mon.0) 112 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049468+0000 mon.a (mon.0) 113 : cluster [ERR] osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049481+0000 mon.a (mon.0) 114 : cluster [ERR] osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049512+0000 mon.a (mon.0) 115 : cluster [ERR] osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049529+0000 mon.a (mon.0) 116 : cluster [ERR] osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049571+0000 mon.a (mon.0) 117 : cluster [ERR] osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:28.995478+0000 mgr.y (mgr.34107) 67 : cluster [DBG] pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:31.301 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.013052+0000 mon.c (mon.2) 1 : cluster [INF] mon.c calling monitor election 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.014768+0000 mon.a (mon.0) 97 : cluster [INF] mon.a calling monitor election 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.015909+0000 mon.b (mon.1) 47 : cluster [INF] mon.b calling monitor election 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.040688+0000 mon.a (mon.0) 98 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047722+0000 mon.a (mon.0) 99 : cluster [DBG] monmap epoch 3 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047751+0000 mon.a (mon.0) 100 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047774+0000 mon.a (mon.0) 101 : cluster [DBG] last_changed 2024-04-13T07:21:29.816921+0000 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047799+0000 mon.a (mon.0) 102 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047844+0000 mon.a (mon.0) 103 : cluster [DBG] min_mon_release 18 (reef) 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047873+0000 mon.a (mon.0) 104 : cluster [DBG] election_strategy: 1 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047891+0000 mon.a (mon.0) 105 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047917+0000 mon.a (mon.0) 106 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.047931+0000 mon.a (mon.0) 107 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.048708+0000 mon.a (mon.0) 108 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:39:31.302 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.048749+0000 mon.a (mon.0) 109 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049192+0000 mon.a (mon.0) 110 : cluster [DBG] mgrmap e37: y(active, since 93s), standbys: x 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049425+0000 mon.a (mon.0) 111 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049454+0000 mon.a (mon.0) 112 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049468+0000 mon.a (mon.0) 113 : cluster [ERR] osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049481+0000 mon.a (mon.0) 114 : cluster [ERR] osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049512+0000 mon.a (mon.0) 115 : cluster [ERR] osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049529+0000 mon.a (mon.0) 116 : cluster [ERR] osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049571+0000 mon.a (mon.0) 117 : cluster [ERR] osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049584+0000 mon.a (mon.0) 118 : cluster [ERR] osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049606+0000 mon.a (mon.0) 119 : cluster [ERR] osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.303 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.049633+0000 mon.a (mon.0) 120 : cluster [ERR] osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: audit 2024-04-13T07:39:30.057997+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: audit 2024-04-13T07:39:30.067733+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[82331]: audit 2024-04-13T07:39:30.108109+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049584+0000 mon.a (mon.0) 118 : cluster [ERR] osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049606+0000 mon.a (mon.0) 119 : cluster [ERR] osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.049633+0000 mon.a (mon.0) 120 : cluster [ERR] osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: audit 2024-04-13T07:39:30.057997+0000 mon.a (mon.0) 121 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: audit 2024-04-13T07:39:30.067733+0000 mon.a (mon.0) 122 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:31.304 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:31 smithi089 bash[84777]: audit 2024-04-13T07:39:30.108109+0000 mon.a (mon.0) 123 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:31.375 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2024-04-13T07:39:31.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:28.995478+0000 mgr.y (mgr.34107) 67 : cluster 0 pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.013052+0000 mon.c (mon.2) 1 : cluster 1 mon.c calling monitor election 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.014768+0000 mon.a (mon.0) 97 : cluster 1 mon.a calling monitor election 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.015909+0000 mon.b (mon.1) 47 : cluster 1 mon.b calling monitor election 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.040688+0000 mon.a (mon.0) 98 : cluster 1 mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047722+0000 mon.a (mon.0) 99 : cluster 0 monmap epoch 3 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047751+0000 mon.a (mon.0) 100 : cluster 0 fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047774+0000 mon.a (mon.0) 101 : cluster 0 last_changed 2024-04-13T07:21:29.816921+0000 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047799+0000 mon.a (mon.0) 102 : cluster 0 created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047844+0000 mon.a (mon.0) 103 : cluster 0 min_mon_release 18 (reef) 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047873+0000 mon.a (mon.0) 104 : cluster 0 election_strategy: 1 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047891+0000 mon.a (mon.0) 105 : cluster 0 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:39:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047917+0000 mon.a (mon.0) 106 : cluster 0 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.047931+0000 mon.a (mon.0) 107 : cluster 0 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.048708+0000 mon.a (mon.0) 108 : cluster 0 fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.048749+0000 mon.a (mon.0) 109 : cluster 0 osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049192+0000 mon.a (mon.0) 110 : cluster 0 mgrmap e37: y(active, since 93s), standbys: x 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049425+0000 mon.a (mon.0) 111 : cluster 4 Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049454+0000 mon.a (mon.0) 112 : cluster 4 [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049468+0000 mon.a (mon.0) 113 : cluster 4 osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049481+0000 mon.a (mon.0) 114 : cluster 4 osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049512+0000 mon.a (mon.0) 115 : cluster 4 osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049529+0000 mon.a (mon.0) 116 : cluster 4 osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049571+0000 mon.a (mon.0) 117 : cluster 4 osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049584+0000 mon.a (mon.0) 118 : cluster 4 osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049606+0000 mon.a (mon.0) 119 : cluster 4 osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.049633+0000 mon.a (mon.0) 120 : cluster 4 osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:31.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: audit 2024-04-13T07:39:30.057997+0000 mon.a (mon.0) 121 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:31.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: audit 2024-04-13T07:39:30.067733+0000 mon.a (mon.0) 122 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:31.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:31 smithi203 bash[23480]: audit 2024-04-13T07:39:30.108109+0000 mon.a (mon.0) 123 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:32.308 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:32 smithi203 bash[23480]: cluster 2024-04-13T07:39:30.996077+0000 mgr.y (mgr.34107) 68 : cluster 0 pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:32.308 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:32 smithi203 bash[23480]: cluster 2024-04-13T07:39:31.075569+0000 mon.a (mon.0) 124 : cluster 0 mgrmap e38: y(active, since 94s), standbys: x 2024-04-13T07:39:32.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:32 smithi089 bash[82331]: cluster 2024-04-13T07:39:30.996077+0000 mgr.y (mgr.34107) 68 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:32.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:32 smithi089 bash[82331]: cluster 2024-04-13T07:39:31.075569+0000 mon.a (mon.0) 124 : cluster [DBG] mgrmap e38: y(active, since 94s), standbys: x 2024-04-13T07:39:32.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:32 smithi089 bash[84777]: cluster 2024-04-13T07:39:30.996077+0000 mgr.y (mgr.34107) 68 : cluster [DBG] pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:32.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:32 smithi089 bash[84777]: cluster 2024-04-13T07:39:31.075569+0000 mon.a (mon.0) 124 : cluster [DBG] mgrmap e38: y(active, since 94s), standbys: x 2024-04-13T07:39:32.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:32 smithi203 bash[60252]: ts=2024-04-13T07:39:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:33.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:33 smithi203 bash[60252]: ts=2024-04-13T07:39:33.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:34.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:34 smithi203 bash[23480]: cluster 2024-04-13T07:39:32.996968+0000 mgr.y (mgr.34107) 69 : cluster 0 pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:34.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:34 smithi089 bash[82331]: cluster 2024-04-13T07:39:32.996968+0000 mgr.y (mgr.34107) 69 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:34.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:34 smithi089 bash[84777]: cluster 2024-04-13T07:39:32.996968+0000 mgr.y (mgr.34107) 69 : cluster [DBG] pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:35.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:34 smithi089 bash[74725]: debug 2024-04-13T07:39:34.806+0000 7faae2853700 -1 mgr.server handle_report got status from non-daemon mon.c 2024-04-13T07:39:36.138 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:39:36.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:35 smithi203 bash[23480]: cluster 2024-04-13T07:39:34.997958+0000 mgr.y (mgr.34107) 70 : cluster 0 pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:36.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:35 smithi089 bash[84777]: cluster 2024-04-13T07:39:34.997958+0000 mgr.y (mgr.34107) 70 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:36.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:35 smithi089 bash[82331]: cluster 2024-04-13T07:39:34.997958+0000 mgr.y (mgr.34107) 70 : cluster [DBG] pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:36.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:39:35] "GET /metrics HTTP/1.1" 200 33884 "" "Prometheus/2.43.0" 2024-04-13T07:39:37.217 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:36 smithi089 bash[84777]: audit 2024-04-13T07:39:35.920944+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:37.218 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:36 smithi089 bash[84777]: audit 2024-04-13T07:39:35.929801+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:37.218 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:36 smithi089 bash[82331]: audit 2024-04-13T07:39:35.920944+0000 mon.a (mon.0) 125 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:37.218 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:36 smithi089 bash[82331]: audit 2024-04-13T07:39:35.929801+0000 mon.a (mon.0) 126 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:37.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:36 smithi203 bash[23480]: audit 2024-04-13T07:39:35.920944+0000 mon.a (mon.0) 125 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:37.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:36 smithi203 bash[23480]: audit 2024-04-13T07:39:35.929801+0000 mon.a (mon.0) 126 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:38.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:37 smithi089 bash[82331]: audit 2024-04-13T07:39:36.982232+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:38.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:37 smithi089 bash[82331]: audit 2024-04-13T07:39:36.987578+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:38.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:37 smithi089 bash[82331]: cluster 2024-04-13T07:39:36.999115+0000 mgr.y (mgr.34107) 71 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:38.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:37 smithi089 bash[84777]: audit 2024-04-13T07:39:36.982232+0000 mon.a (mon.0) 127 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:38.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:37 smithi089 bash[84777]: audit 2024-04-13T07:39:36.987578+0000 mon.a (mon.0) 128 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:38.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:37 smithi089 bash[84777]: cluster 2024-04-13T07:39:36.999115+0000 mgr.y (mgr.34107) 71 : cluster [DBG] pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:38.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:37 smithi203 bash[23480]: audit 2024-04-13T07:39:36.982232+0000 mon.a (mon.0) 127 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:38.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:37 smithi203 bash[23480]: audit 2024-04-13T07:39:36.987578+0000 mon.a (mon.0) 128 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:38.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:37 smithi203 bash[23480]: cluster 2024-04-13T07:39:36.999115+0000 mgr.y (mgr.34107) 71 : cluster 0 pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:38.887 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev)": 2 2024-04-13T07:39:39.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:38 smithi203 bash[23480]: audit 2024-04-13T07:39:38.880718+0000 mon.c (mon.2) 2 : audit 0 from='client.? 172.21.15.89:0/1217145085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:39:39.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:38 smithi089 bash[82331]: audit 2024-04-13T07:39:38.880718+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.89:0/1217145085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:39:39.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:38 smithi089 bash[84777]: audit 2024-04-13T07:39:38.880718+0000 mon.c (mon.2) 2 : audit [DBG] from='client.? 172.21.15.89:0/1217145085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:39:39.503 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch ps' 2024-04-13T07:39:40.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:40 smithi203 bash[23480]: cluster 2024-04-13T07:39:38.999702+0000 mgr.y (mgr.34107) 72 : cluster 0 pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:40.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:40 smithi089 bash[82331]: cluster 2024-04-13T07:39:38.999702+0000 mgr.y (mgr.34107) 72 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:40.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:40 smithi089 bash[84777]: cluster 2024-04-13T07:39:38.999702+0000 mgr.y (mgr.34107) 72 : cluster [DBG] pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2024-04-13T07:39:42.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:42 smithi089 bash[84777]: cluster 2024-04-13T07:39:41.000327+0000 mgr.y (mgr.34107) 73 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:42 smithi089 bash[82331]: cluster 2024-04-13T07:39:41.000327+0000 mgr.y (mgr.34107) 73 : cluster [DBG] pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:42.307 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:42 smithi203 bash[23480]: cluster 2024-04-13T07:39:41.000327+0000 mgr.y (mgr.34107) 73 : cluster 0 pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:42.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:42 smithi203 bash[60252]: ts=2024-04-13T07:39:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:43.278 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:43 smithi089 bash[82331]: audit 2024-04-13T07:39:42.060147+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:39:43.278 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:43 smithi089 bash[84777]: audit 2024-04-13T07:39:42.060147+0000 mon.a (mon.0) 129 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:39:43.501 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:43 smithi203 bash[23480]: audit 2024-04-13T07:39:42.060147+0000 mon.a (mon.0) 129 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:39:43.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:43 smithi203 bash[60252]: ts=2024-04-13T07:39:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:44.202 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:39:44.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:44 smithi203 bash[23480]: cluster 2024-04-13T07:39:43.000886+0000 mgr.y (mgr.34107) 74 : cluster 0 pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:44.590 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:44 smithi089 bash[82331]: cluster 2024-04-13T07:39:43.000886+0000 mgr.y (mgr.34107) 74 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:44.590 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:44 smithi089 bash[84777]: cluster 2024-04-13T07:39:43.000886+0000 mgr.y (mgr.34107) 74 : cluster [DBG] pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:46.141 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:39:45] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.43.0" 2024-04-13T07:39:46.141 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:46 smithi089 bash[84777]: cluster 2024-04-13T07:39:45.001396+0000 mgr.y (mgr.34107) 75 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:46.141 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:46 smithi089 bash[82331]: cluster 2024-04-13T07:39:45.001396+0000 mgr.y (mgr.34107) 75 : cluster [DBG] pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:46.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:46 smithi203 bash[23480]: cluster 2024-04-13T07:39:45.001396+0000 mgr.y (mgr.34107) 75 : cluster 0 pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:47.695 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-13T07:39:47.695 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (9m) 11s ago 9m 13.5M - 0.25.0 c8568f914cd2 d39c47986efe 2024-04-13T07:39:47.695 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi203 *:3000 running (9m) 25s ago 9m 78.3M - 9.4.7 954c08fa6188 4496ec520de5 2024-04-13T07:39:47.695 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.pccnvq smithi089 running (7m) 11s ago 7m 14.1M - 18.2.2-1217-g7d36322c 6ecd1226aa2f 07167bfc6e5f 2024-04-13T07:39:47.695 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi203.sjspdy smithi203 running (7m) 25s ago 7m 16.1M - 18.2.2-1217-g7d36322c 6ecd1226aa2f e100a52f8447 2024-04-13T07:39:47.695 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi203 *:8443,9283,8765 running (2m) 25s ago 17m 436M - 19.0.0-2905-g5f22a497 c4b45bf0c946 54a5a6c61bc7 2024-04-13T07:39:47.695 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (3m) 11s ago 20m 485M - 19.0.0-2905-g5f22a497 c4b45bf0c946 f20e719fbf68 2024-04-13T07:39:47.695 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (2m) 11s ago 20m 40.9M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 d2e86a9f9ec2 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi203 running (18m) 25s ago 18m 46.1M 2048M 18.2.2-1217-g7d36322c 6ecd1226aa2f e475d5b92fda 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (19s) 11s ago 18m 18.6M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 0345f038f9be 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (9m) 11s ago 10m 8479k - 1.5.0 0da6a335fe13 04462903d931 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi203 *:9100 running (9m) 25s ago 9m 8676k - 1.5.0 0da6a335fe13 a79084a2b37c 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (17m) 11s ago 17m 60.4M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 599c6c3ff3d6 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (16m) 11s ago 16m 52.1M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 58fb3beb85cc 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (15m) 11s ago 15m 50.6M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f e020a590fd83 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (14m) 11s ago 14m 47.0M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 1c6772886276 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi203 running (13m) 25s ago 13m 55.5M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 28af48f1a163 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi203 running (13m) 25s ago 13m 48.2M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f afe79fa4d7d0 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi203 running (12m) 25s ago 12m 47.4M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 8913a43d05b8 2024-04-13T07:39:47.696 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi203 running (11m) 25s ago 11m 62.4M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 01ed33979f80 2024-04-13T07:39:47.697 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi203 *:9095 running (2m) 25s ago 10m 48.1M - 2.43.0 a07b618ecd1d 50a6268920e0 2024-04-13T07:39:48.367 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch upgrade pause' 2024-04-13T07:39:48.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: cluster 2024-04-13T07:39:47.001940+0000 mgr.y (mgr.34107) 76 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:48.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.291535+0000 mon.a (mon.0) 130 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.299512+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.300947+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.301987+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.307241+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.366721+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.368793+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.370025+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.370696+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: cephadm 2024-04-13T07:39:47.371250+0000 mgr.y (mgr.34107) 77 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[82331]: audit 2024-04-13T07:39:47.678293+0000 mgr.y (mgr.34107) 78 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: cluster 2024-04-13T07:39:47.001940+0000 mgr.y (mgr.34107) 76 : cluster [DBG] pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:48.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.291535+0000 mon.a (mon.0) 130 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.299512+0000 mon.a (mon.0) 131 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.300947+0000 mon.a (mon.0) 132 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.301987+0000 mon.a (mon.0) 133 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.307241+0000 mon.a (mon.0) 134 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.366721+0000 mon.a (mon.0) 135 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.368793+0000 mon.a (mon.0) 136 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.370025+0000 mon.a (mon.0) 137 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.370696+0000 mon.a (mon.0) 138 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: cephadm 2024-04-13T07:39:47.371250+0000 mgr.y (mgr.34107) 77 : cephadm [INF] Upgrade: It appears safe to stop mon.b 2024-04-13T07:39:48.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:48 smithi089 bash[84777]: audit 2024-04-13T07:39:47.678293+0000 mgr.y (mgr.34107) 78 : audit [DBG] from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:39:48.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: cluster 2024-04-13T07:39:47.001940+0000 mgr.y (mgr.34107) 76 : cluster 0 pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:48.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.291535+0000 mon.a (mon.0) 130 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.299512+0000 mon.a (mon.0) 131 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.300947+0000 mon.a (mon.0) 132 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:48.745 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.301987+0000 mon.a (mon.0) 133 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:39:48.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.307241+0000 mon.a (mon.0) 134 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:48.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.366721+0000 mon.a (mon.0) 135 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:39:48.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.368793+0000 mon.a (mon.0) 136 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:39:48.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.370025+0000 mon.a (mon.0) 137 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2024-04-13T07:39:48.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.370696+0000 mon.a (mon.0) 138 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2024-04-13T07:39:48.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: cephadm 2024-04-13T07:39:47.371250+0000 mgr.y (mgr.34107) 77 : cephadm 1 Upgrade: It appears safe to stop mon.b 2024-04-13T07:39:48.746 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:48 smithi203 bash[23480]: audit 2024-04-13T07:39:47.678293+0000 mgr.y (mgr.34107) 78 : audit 0 from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[82331]: cluster 2024-04-13T07:39:49.003070+0000 mgr.y (mgr.34107) 79 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[82331]: audit 2024-04-13T07:39:49.849375+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[82331]: audit 2024-04-13T07:39:49.853528+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[82331]: audit 2024-04-13T07:39:49.855192+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[82331]: audit 2024-04-13T07:39:49.856865+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[84777]: cluster 2024-04-13T07:39:49.003070+0000 mgr.y (mgr.34107) 79 : cluster [DBG] pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[84777]: audit 2024-04-13T07:39:49.849375+0000 mon.a (mon.0) 139 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[84777]: audit 2024-04-13T07:39:49.853528+0000 mon.a (mon.0) 140 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:39:50.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[84777]: audit 2024-04-13T07:39:49.855192+0000 mon.a (mon.0) 141 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:39:50.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:50 smithi089 bash[84777]: audit 2024-04-13T07:39:49.856865+0000 mon.a (mon.0) 142 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:50.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:50 smithi203 bash[23480]: cluster 2024-04-13T07:39:49.003070+0000 mgr.y (mgr.34107) 79 : cluster 0 pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:50.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:50 smithi203 bash[23480]: audit 2024-04-13T07:39:49.849375+0000 mon.a (mon.0) 139 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:39:50.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:50 smithi203 bash[23480]: audit 2024-04-13T07:39:49.853528+0000 mon.a (mon.0) 140 : audit 1 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:39:50.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:50 smithi203 bash[23480]: audit 2024-04-13T07:39:49.855192+0000 mon.a (mon.0) 141 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:39:50.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:50 smithi203 bash[23480]: audit 2024-04-13T07:39:49.856865+0000 mon.a (mon.0) 142 : audit 0 from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:39:51.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:51 smithi203 bash[23480]: cephadm 2024-04-13T07:39:49.842409+0000 mgr.y (mgr.34107) 80 : cephadm 1 Upgrade: Updating mon.b 2024-04-13T07:39:51.505 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:51 smithi203 bash[23480]: cephadm 2024-04-13T07:39:49.858882+0000 mgr.y (mgr.34107) 81 : cephadm 1 Deploying daemon mon.b on smithi203 2024-04-13T07:39:51.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:51 smithi089 bash[82331]: cephadm 2024-04-13T07:39:49.842409+0000 mgr.y (mgr.34107) 80 : cephadm [INF] Upgrade: Updating mon.b 2024-04-13T07:39:51.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:51 smithi089 bash[82331]: cephadm 2024-04-13T07:39:49.858882+0000 mgr.y (mgr.34107) 81 : cephadm [INF] Deploying daemon mon.b on smithi203 2024-04-13T07:39:51.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:51 smithi089 bash[84777]: cephadm 2024-04-13T07:39:49.842409+0000 mgr.y (mgr.34107) 80 : cephadm [INF] Upgrade: Updating mon.b 2024-04-13T07:39:51.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:51 smithi089 bash[84777]: cephadm 2024-04-13T07:39:49.858882+0000 mgr.y (mgr.34107) 81 : cephadm [INF] Deploying daemon mon.b on smithi203 2024-04-13T07:39:52.266 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.267 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.267 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.267 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.268 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: Stopping Ceph mon.b for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:39:52.268 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.269 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.269 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:39:52 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:52.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:52 smithi089 bash[82331]: cluster 2024-04-13T07:39:51.003690+0000 mgr.y (mgr.34107) 82 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:52.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:39:52 smithi089 bash[84777]: cluster 2024-04-13T07:39:51.003690+0000 mgr.y (mgr.34107) 82 : cluster [DBG] pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:52.680 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:52 smithi203 bash[23480]: cluster 2024-04-13T07:39:51.003690+0000 mgr.y (mgr.34107) 82 : cluster 0 pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:39:52.680 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:52 smithi203 bash[23480]: debug 2024-04-13T07:39:52.391+0000 7fae3ad25700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-13T07:39:52.680 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:52 smithi203 bash[23480]: debug 2024-04-13T07:39:52.391+0000 7fae3ad25700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2024-04-13T07:39:52.680 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:52 smithi203 bash[60252]: ts=2024-04-13T07:39:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:53.111 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:39:53.455 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:53 smithi203 bash[67943]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mon-b 2024-04-13T07:39:53.455 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:53 smithi203 bash[68004]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-mon-b 2024-04-13T07:39:53.455 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.b.service: Deactivated successfully. 2024-04-13T07:39:53.455 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: Stopped Ceph mon.b for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:39:53.764 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.764 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.765 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.765 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.765 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: Started Ceph mon.b for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:39:53.766 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.766 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.767 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:53 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:39:53.767 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:39:53 smithi203 bash[60252]: ts=2024-04-13T07:39:53.506Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:39:55.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.711+0000 7faec17bdc80 0 set uid:gid to 167:167 (ceph:ceph) 2024-04-13T07:39:55.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.711+0000 7faec17bdc80 0 ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev), process ceph-mon, pid 7 2024-04-13T07:39:55.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.711+0000 7faec17bdc80 0 pidfile_write: ignore empty --pid-file 2024-04-13T07:39:55.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 0 load: jerasure load: lrc 2024-04-13T07:39:55.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: RocksDB version: 7.9.2 2024-04-13T07:39:55.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Git sha 0 2024-04-13T07:39:55.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Compile date 2024-04-12 15:38:20 2024-04-13T07:39:55.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: DB SUMMARY 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: DB Session ID: QMS3Z0HFT03JBR1NQAQI 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: CURRENT file: CURRENT 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: IDENTITY file: IDENTITY 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: MANIFEST file: MANIFEST-000010 size: 2120 Bytes 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000036.sst 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000034.log size: 1393733 ; 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.error_if_exists: 0 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.create_if_missing: 0 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.paranoid_checks: 1 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.flush_verify_memtable_count: 1 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.track_and_verify_wals_in_manifest: 0 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.env: 0x555a270e2cc0 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.fs: PosixFileSystem 2024-04-13T07:39:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.info_log: 0x555a27ea7c40 2024-04-13T07:39:55.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_file_opening_threads: 16 2024-04-13T07:39:55.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.statistics: (nil) 2024-04-13T07:39:55.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.use_fsync: 0 2024-04-13T07:39:55.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_log_file_size: 0 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_manifest_file_size: 1073741824 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.log_file_time_to_roll: 0 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.keep_log_file_num: 1000 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.recycle_log_file_num: 0 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.allow_fallocate: 1 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.allow_mmap_reads: 0 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.allow_mmap_writes: 0 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.use_direct_reads: 0 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.create_missing_column_families: 0 2024-04-13T07:39:55.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.db_log_dir: 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.wal_dir: 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.table_cache_numshardbits: 6 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.WAL_ttl_seconds: 0 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.WAL_size_limit_MB: 0 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.manifest_preallocation_size: 4194304 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.is_fd_close_on_exec: 1 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.advise_random_on_open: 1 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.db_write_buffer_size: 0 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.write_buffer_manager: 0x555a27e7a820 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.access_hint_on_compaction_start: 1 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.random_access_max_buffer_size: 1048576 2024-04-13T07:39:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.use_adaptive_mutex: 0 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.rate_limiter: (nil) 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.wal_recovery_mode: 2 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.enable_thread_tracking: 0 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.enable_pipelined_write: 0 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.unordered_write: 0 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.allow_concurrent_memtable_write: 1 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.enable_write_thread_adaptive_yield: 1 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.write_thread_max_yield_usec: 100 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.write_thread_slow_yield_usec: 3 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.row_cache: None 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.wal_filter: None 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.avoid_flush_during_recovery: 0 2024-04-13T07:39:55.020 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.allow_ingest_behind: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.two_write_queues: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.manual_wal_flush: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.wal_compression: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.atomic_flush: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.avoid_unnecessary_blocking_io: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.persist_stats_to_disk: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.write_dbid_to_manifest: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.log_readahead_size: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.file_checksum_gen_factory: Unknown 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.best_efforts_recovery: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bgerror_resume_count: 2147483647 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bgerror_resume_retry_interval: 1000000 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.allow_data_in_errors: 0 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.db_host_id: __hostname__ 2024-04-13T07:39:55.021 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.enforce_single_del_contracts: true 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_background_jobs: 2 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_background_compactions: -1 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_subcompactions: 1 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.avoid_flush_during_shutdown: 0 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.writable_file_max_buffer_size: 1048576 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.delayed_write_rate : 16777216 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_total_wal_size: 0 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.stats_dump_period_sec: 600 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.stats_persist_period_sec: 600 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.stats_history_buffer_size: 1048576 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_open_files: -1 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bytes_per_sync: 0 2024-04-13T07:39:55.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.wal_bytes_per_sync: 0 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.strict_bytes_per_sync: 0 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_readahead_size: 0 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_background_flushes: -1 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Compression algorithms supported: 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: kZSTDNotFinalCompression supported: 0 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: kZSTD supported: 0 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: kXpressCompression supported: 0 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: kLZ4HCCompression supported: 1 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: kZlibCompression supported: 1 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: kSnappyCompression supported: 1 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: kLZ4Compression supported: 1 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: kBZip2Compression supported: 0 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Fast CRC32 supported: Supported on x86 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: DMutex implementation: pthread_mutex_t 2024-04-13T07:39:55.023 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.comparator: leveldb.BytewiseComparator 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.merge_operator: 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_filter: None 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_filter_factory: None 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.sst_partitioner_factory: None 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.memtable_factory: SkipListFactory 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.table_factory: BlockBasedTable 2024-04-13T07:39:55.025 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x555a27ea7e00) 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: cache_index_and_filter_blocks: 1 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: cache_index_and_filter_blocks_with_high_priority: 0 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: pin_l0_filter_and_index_blocks_in_cache: 0 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: pin_top_level_index_and_filter: 1 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: index_type: 0 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: data_block_index_type: 0 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: index_shortening: 1 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: data_block_hash_table_util_ratio: 0.750000 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: checksum: 4 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: no_block_cache: 0 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: block_cache: 0x555a27eab090 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: block_cache_name: BinnedLRUCache 2024-04-13T07:39:55.026 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: block_cache_options: 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: capacity : 536870912 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: num_shard_bits : 4 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: strict_capacity_limit : 0 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: high_pri_pool_ratio: 0.000 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: block_cache_compressed: (nil) 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: persistent_cache: (nil) 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: block_size: 4096 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: block_size_deviation: 10 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: block_restart_interval: 16 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: index_block_restart_interval: 1 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: metadata_block_size: 4096 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: partition_filters: 0 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: use_delta_encoding: 1 2024-04-13T07:39:55.027 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: filter_policy: bloomfilter 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: whole_key_filtering: 1 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: verify_compression: 0 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: read_amp_bytes_per_bit: 0 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: format_version: 5 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: enable_index_compression: 1 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: block_align: 0 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: max_auto_readahead_size: 262144 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: prepopulate_block_cache: 0 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: initial_auto_readahead_size: 8192 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: num_file_reads_for_auto_readahead: 2 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.write_buffer_size: 33554432 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_write_buffer_number: 2 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression: NoCompression 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression: Disabled 2024-04-13T07:39:55.028 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.prefix_extractor: nullptr 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.num_levels: 7 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.min_write_buffer_number_to_merge: 1 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_write_buffer_number_to_maintain: 0 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_write_buffer_size_to_maintain: 0 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.window_bits: -14 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.level: 32767 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.strategy: 0 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.enabled: false 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:39:55.029 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.window_bits: -14 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.level: 32767 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.strategy: 0 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.max_dict_bytes: 0 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.parallel_threads: 1 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.enabled: false 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.level0_file_num_compaction_trigger: 4 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.level0_slowdown_writes_trigger: 20 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.level0_stop_writes_trigger: 36 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.target_file_size_base: 67108864 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.target_file_size_multiplier: 1 2024-04-13T07:39:55.030 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_base: 268435456 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_sequential_skip_in_iterations: 8 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_compaction_bytes: 1677721600 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.arena_block_size: 1048576 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2024-04-13T07:39:55.031 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.disable_auto_compactions: 0 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_style: kCompactionStyleLevel 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_pri: kMinOverlappingRatio 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_options_universal.size_ratio: 1 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_options_universal.min_merge_width: 2 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.table_properties_collectors: CompactOnDeletionCollector (Sliding window size = 32768 Deletion trigger = 16384 Deletion ratio = 0); 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.inplace_update_support: 0 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.inplace_update_num_locks: 10000 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2024-04-13T07:39:55.032 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.memtable_whole_key_filtering: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.memtable_huge_page_size: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.bloom_locality: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.max_successive_merges: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.optimize_filters_for_hits: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.paranoid_file_checks: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.force_consistency_checks: 1 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.report_bg_io_stats: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.ttl: 2592000 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.periodic_compaction_seconds: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.preclude_last_level_data_seconds: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.preserve_internal_time_seconds: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.enable_blob_files: false 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.min_blob_size: 0 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.blob_file_size: 268435456 2024-04-13T07:39:55.033 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.blob_compression_type: NoCompression 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.enable_blob_garbage_collection: false 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.blob_compaction_readahead_size: 0 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.blob_file_starting_level: 0 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: Options.experimental_mempurge_threshold: 0.000000 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000010 succeeded,manifest_file_number is 10, next_file_number is 38, last_sequence is 18376, log_number is 34,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 34 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 34 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 521ff828-bf84-4d01-bb76-34506c04be8c 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993994722788, "job": 1, "event": "recovery_started", "wal_files": [34]} 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.719+0000 7faec17bdc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #34 mode 2 2024-04-13T07:39:55.034 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.727+0000 7faec17bdc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993994728196, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 39, "file_size": 865933, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 18381, "largest_seqno": 19134, "table_properties": {"data_size": 862094, "index_size": 2031, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 773, "raw_key_size": 6967, "raw_average_key_size": 23, "raw_value_size": 855462, "raw_average_value_size": 2832, "num_data_blocks": 92, "num_entries": 302, "num_filter_entries": 302, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[CompactOnDeletionCollector]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1712993994, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "521ff828-bf84-4d01-bb76-34506c04be8c", "db_session_id": "QMS3Z0HFT03JBR1NQAQI", "orig_file_number": 39, "seqno_to_time_mapping": "N/A"}} 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.727+0000 7faec17bdc80 4 rocksdb: EVENT_LOG_v1 {"time_micros": 1712993994728305, "job": 1, "event": "recovery_finished"} 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.727+0000 7faec17bdc80 4 rocksdb: [db/version_set.cc:5047] Creating manifest 41 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.727+0000 7faec17bdc80 4 rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 4 rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000034.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 4 rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x555a27f9e000 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 4 rocksdb: DB pointer 0x555a27f88000 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faeaff17700 4 rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faeaff17700 4 rocksdb: [db/db_impl/db_impl.cc:1111] 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: ** DB Stats ** 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2024-04-13T07:39:55.035 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: ** Compaction Stats [default] ** 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: L0 1/0 845.64 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 314.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: L6 1/0 8.35 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Sum 2/0 9.17 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 314.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 314.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: ** Compaction Stats [default] ** 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 314.4 0.00 0.00 1 0.003 0 0 0.0 0.0 2024-04-13T07:39:55.036 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Uptime(secs): 0.0 total, 0.0 interval 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Flush(GB): cumulative 0.001, interval 0.001 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: AddFile(GB): cumulative 0.000, interval 0.000 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: AddFile(Total Files): cumulative 0, interval 0 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: AddFile(L0 Files): cumulative 0, interval 0 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: AddFile(Keys): cumulative 0, interval 0 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Cumulative compaction: 0.00 GB write, 80.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Interval compaction: 0.00 GB write, 80.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Block cache BinnedLRUCache@0x555a27eab090#7 capacity: 512.00 MB usage: 2.98 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Block cache entry stats(count,size,portion): FilterBlock(1,0.91 KB,0.000172853%) IndexBlock(1,2.08 KB,0.000396371%) Misc(1,0.00 KB,0%) 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: ** File Read Latency Histogram By Level [default] ** 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 0 starting mon.b rank 1 at public addrs [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] at bind addrs [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:39:55.037 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 1 mon.b@-1(???) e3 preinit fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 0 mon.b@-1(???).mds e10 new map 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 0 mon.b@-1(???).mds e10 print_map 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: e10 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: enable_multiple, ever_enabled_multiple: 1,1 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: legacy client fscid: 1 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Filesystem 'foo' (1) 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: fs_name foo 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: epoch 10 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: flags 12 joinable allow_snaps allow_multimds_snaps 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: created 2024-04-13T07:32:03.054183+0000 2024-04-13T07:39:55.038 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: modified 2024-04-13T07:32:13.856309+0000 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: tableserver 0 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: root 0 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: session_timeout 60 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: session_autoclose 300 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: max_file_size 1099511627776 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: max_xattr_size 65536 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: required_client_features {} 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: last_failure 0 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: last_failure_osd_epoch 70 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: max_mds 1 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: in 0 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: up {0=24364} 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: failed 2024-04-13T07:39:55.039 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: damaged 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: stopped 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: data_pools [4] 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: metadata_pool 3 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: inline_data disabled 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: balancer 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: bal_rank_mask -1 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: standby_count_wanted 1 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: qdb_cluster leader: 0 members: 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: [mds.foo.smithi203.sjspdy{0:24364} state up:active seq 4 join_fscid=1 addr [v2:172.21.15.203:6832/3082035284,v1:172.21.15.203:6833/3082035284] compat {c=[1],r=[1],i=[7ff]}] 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: Standby daemons: 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: 2024-04-13T07:39:55.040 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: [mds.foo.smithi089.pccnvq{-1:14583} state up:standby seq 1 join_fscid=1 addr [v2:172.21.15.89:6834/2537258916,v1:172.21.15.89:6835/2537258916] compat {c=[1],r=[1],i=[7ff]}] 2024-04-13T07:39:55.041 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 0 mon.b@-1(???).osd e92 crush map has features 3314933000854323200, adjusting msgr requires 2024-04-13T07:39:55.041 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 0 mon.b@-1(???).osd e92 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:39:55.041 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 0 mon.b@-1(???).osd e92 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:39:55.041 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 0 mon.b@-1(???).osd e92 crush map has features 432629239337189376, adjusting msgr requires 2024-04-13T07:39:55.041 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:39:54 smithi203 bash[68060]: debug 2024-04-13T07:39:54.731+0000 7faec17bdc80 1 mon.b@-1(???).paxosservice(auth 1..23) refresh upgraded, format 0 -> 3 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:55.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[82331]: debug 2024-04-13T07:39:55.633+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:39:56.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:39:55] "GET /metrics HTTP/1.1" 200 33881 "" "Prometheus/2.43.0" 2024-04-13T07:40:00.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:39:59 smithi089 bash[74725]: debug 2024-04-13T07:39:59.737+0000 7faae2853700 -1 mgr.server handle_report got status from non-daemon mon.b 2024-04-13T07:40:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:40:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:40:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.693+0000 7f9605e44700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:00 smithi089 bash[82331]: debug 2024-04-13T07:40:00.757+0000 7f960363f700 -1 log_channel(cluster) log [ERR] : osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:01.873 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:39:55.664334+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:39:55.685472+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.690474+0000 mon.a (mon.0) 174 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.690500+0000 mon.a (mon.0) 175 : cluster [DBG] last_changed 2024-04-13T07:39:55.634748+0000 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.690521+0000 mon.a (mon.0) 176 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.690541+0000 mon.a (mon.0) 177 : cluster [DBG] min_mon_release 19 (squid) 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.690571+0000 mon.a (mon.0) 178 : cluster [DBG] election_strategy: 1 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.690596+0000 mon.a (mon.0) 179 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:40:02.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.690619+0000 mon.a (mon.0) 180 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.690647+0000 mon.a (mon.0) 181 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.691769+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.691829+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.692381+0000 mon.a (mon.0) 184 : cluster [DBG] mgrmap e38: y(active, since 2m), standbys: x 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.692692+0000 mon.a (mon.0) 185 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.697824+0000 mon.a (mon.0) 186 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.697864+0000 mon.a (mon.0) 187 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.697922+0000 mon.a (mon.0) 188 : cluster [ERR] osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.697981+0000 mon.a (mon.0) 189 : cluster [ERR] osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.698031+0000 mon.a (mon.0) 190 : cluster [ERR] osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.698071+0000 mon.a (mon.0) 191 : cluster [ERR] osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.698130+0000 mon.a (mon.0) 192 : cluster [ERR] osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.698165+0000 mon.a (mon.0) 193 : cluster [ERR] osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.698205+0000 mon.a (mon.0) 194 : cluster [ERR] osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.698236+0000 mon.a (mon.0) 195 : cluster [ERR] osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.698830+0000 mon.a (mon.0) 196 : cluster [INF] mon.a calling monitor election 2024-04-13T07:40:02.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.720869+0000 mon.a (mon.0) 197 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:40:02.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.734862+0000 mon.a (mon.0) 198 : cluster [DBG] monmap epoch 4 2024-04-13T07:40:02.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.734895+0000 mon.a (mon.0) 199 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.734915+0000 mon.a (mon.0) 200 : cluster [DBG] last_changed 2024-04-13T07:39:55.634748+0000 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.734933+0000 mon.a (mon.0) 201 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.734975+0000 mon.a (mon.0) 202 : cluster [DBG] min_mon_release 19 (squid) 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.735063+0000 mon.a (mon.0) 203 : cluster [DBG] election_strategy: 1 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.735087+0000 mon.a (mon.0) 204 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.735109+0000 mon.a (mon.0) 205 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.735133+0000 mon.a (mon.0) 206 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.736539+0000 mon.a (mon.0) 207 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.736597+0000 mon.a (mon.0) 208 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.737320+0000 mon.a (mon.0) 209 : cluster [DBG] mgrmap e38: y(active, since 2m), standbys: x 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.737640+0000 mon.a (mon.0) 210 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-13T07:40:02.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761378+0000 mon.a (mon.0) 211 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761445+0000 mon.a (mon.0) 212 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761483+0000 mon.a (mon.0) 213 : cluster [ERR] osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761525+0000 mon.a (mon.0) 214 : cluster [ERR] osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761567+0000 mon.a (mon.0) 215 : cluster [ERR] osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761604+0000 mon.a (mon.0) 216 : cluster [ERR] osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761637+0000 mon.a (mon.0) 217 : cluster [ERR] osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761678+0000 mon.a (mon.0) 218 : cluster [ERR] osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761732+0000 mon.a (mon.0) 219 : cluster [ERR] osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:00.761773+0000 mon.a (mon.0) 220 : cluster [ERR] osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: audit 2024-04-13T07:40:00.770408+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: audit 2024-04-13T07:40:00.791780+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: audit 2024-04-13T07:40:00.805461+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:40:02.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:01 smithi203 bash[68060]: cluster 2024-04-13T07:40:01.008181+0000 mgr.y (mgr.34107) 87 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:02.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:39:55.664334+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-04-13T07:40:02.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:39:55.685472+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-04-13T07:40:02.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.690474+0000 mon.a (mon.0) 174 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:40:02.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.690500+0000 mon.a (mon.0) 175 : cluster [DBG] last_changed 2024-04-13T07:39:55.634748+0000 2024-04-13T07:40:02.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.690521+0000 mon.a (mon.0) 176 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:40:02.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.690541+0000 mon.a (mon.0) 177 : cluster [DBG] min_mon_release 19 (squid) 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.690571+0000 mon.a (mon.0) 178 : cluster [DBG] election_strategy: 1 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.690596+0000 mon.a (mon.0) 179 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.690619+0000 mon.a (mon.0) 180 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.690647+0000 mon.a (mon.0) 181 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.691769+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.691829+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.692381+0000 mon.a (mon.0) 184 : cluster [DBG] mgrmap e38: y(active, since 2m), standbys: x 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.692692+0000 mon.a (mon.0) 185 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.697824+0000 mon.a (mon.0) 186 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.697864+0000 mon.a (mon.0) 187 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.697922+0000 mon.a (mon.0) 188 : cluster [ERR] osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.697981+0000 mon.a (mon.0) 189 : cluster [ERR] osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.698031+0000 mon.a (mon.0) 190 : cluster [ERR] osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.698071+0000 mon.a (mon.0) 191 : cluster [ERR] osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.698130+0000 mon.a (mon.0) 192 : cluster [ERR] osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.698165+0000 mon.a (mon.0) 193 : cluster [ERR] osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.698205+0000 mon.a (mon.0) 194 : cluster [ERR] osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:39:55.664334+0000 mon.c (mon.2) 3 : cluster [INF] mon.c calling monitor election 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:39:55.685472+0000 mon.b (mon.1) 2 : cluster [INF] mon.b calling monitor election 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.690474+0000 mon.a (mon.0) 174 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.690500+0000 mon.a (mon.0) 175 : cluster [DBG] last_changed 2024-04-13T07:39:55.634748+0000 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.690521+0000 mon.a (mon.0) 176 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:40:02.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.690541+0000 mon.a (mon.0) 177 : cluster [DBG] min_mon_release 19 (squid) 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.690571+0000 mon.a (mon.0) 178 : cluster [DBG] election_strategy: 1 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.690596+0000 mon.a (mon.0) 179 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.690619+0000 mon.a (mon.0) 180 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.690647+0000 mon.a (mon.0) 181 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.691769+0000 mon.a (mon.0) 182 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.691829+0000 mon.a (mon.0) 183 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.692381+0000 mon.a (mon.0) 184 : cluster [DBG] mgrmap e38: y(active, since 2m), standbys: x 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.692692+0000 mon.a (mon.0) 185 : cluster [WRN] Health check failed: 1/3 mons down, quorum a,c (MON_DOWN) 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.697824+0000 mon.a (mon.0) 186 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.697864+0000 mon.a (mon.0) 187 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.697922+0000 mon.a (mon.0) 188 : cluster [ERR] osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.697981+0000 mon.a (mon.0) 189 : cluster [ERR] osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.698031+0000 mon.a (mon.0) 190 : cluster [ERR] osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.698071+0000 mon.a (mon.0) 191 : cluster [ERR] osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.698130+0000 mon.a (mon.0) 192 : cluster [ERR] osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.698165+0000 mon.a (mon.0) 193 : cluster [ERR] osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.053 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.698205+0000 mon.a (mon.0) 194 : cluster [ERR] osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.698236+0000 mon.a (mon.0) 195 : cluster [ERR] osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.698830+0000 mon.a (mon.0) 196 : cluster [INF] mon.a calling monitor election 2024-04-13T07:40:02.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.720869+0000 mon.a (mon.0) 197 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:40:02.053 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.734862+0000 mon.a (mon.0) 198 : cluster [DBG] monmap epoch 4 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.734895+0000 mon.a (mon.0) 199 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.734915+0000 mon.a (mon.0) 200 : cluster [DBG] last_changed 2024-04-13T07:39:55.634748+0000 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.734933+0000 mon.a (mon.0) 201 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.734975+0000 mon.a (mon.0) 202 : cluster [DBG] min_mon_release 19 (squid) 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.735063+0000 mon.a (mon.0) 203 : cluster [DBG] election_strategy: 1 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.735087+0000 mon.a (mon.0) 204 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.735109+0000 mon.a (mon.0) 205 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.735133+0000 mon.a (mon.0) 206 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.736539+0000 mon.a (mon.0) 207 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.736597+0000 mon.a (mon.0) 208 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.737320+0000 mon.a (mon.0) 209 : cluster [DBG] mgrmap e38: y(active, since 2m), standbys: x 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.737640+0000 mon.a (mon.0) 210 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-13T07:40:02.054 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761378+0000 mon.a (mon.0) 211 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761445+0000 mon.a (mon.0) 212 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761483+0000 mon.a (mon.0) 213 : cluster [ERR] osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761525+0000 mon.a (mon.0) 214 : cluster [ERR] osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761567+0000 mon.a (mon.0) 215 : cluster [ERR] osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761604+0000 mon.a (mon.0) 216 : cluster [ERR] osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761637+0000 mon.a (mon.0) 217 : cluster [ERR] osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761678+0000 mon.a (mon.0) 218 : cluster [ERR] osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761732+0000 mon.a (mon.0) 219 : cluster [ERR] osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:00.761773+0000 mon.a (mon.0) 220 : cluster [ERR] osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: audit 2024-04-13T07:40:00.770408+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: audit 2024-04-13T07:40:00.791780+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: audit 2024-04-13T07:40:00.805461+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:40:02.055 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[84777]: cluster 2024-04-13T07:40:01.008181+0000 mgr.y (mgr.34107) 87 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:02.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.698236+0000 mon.a (mon.0) 195 : cluster [ERR] osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.698830+0000 mon.a (mon.0) 196 : cluster [INF] mon.a calling monitor election 2024-04-13T07:40:02.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.720869+0000 mon.a (mon.0) 197 : cluster [INF] mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2024-04-13T07:40:02.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.734862+0000 mon.a (mon.0) 198 : cluster [DBG] monmap epoch 4 2024-04-13T07:40:02.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.734895+0000 mon.a (mon.0) 199 : cluster [DBG] fsid ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:40:02.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.734915+0000 mon.a (mon.0) 200 : cluster [DBG] last_changed 2024-04-13T07:39:55.634748+0000 2024-04-13T07:40:02.056 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.734933+0000 mon.a (mon.0) 201 : cluster [DBG] created 2024-04-13T07:19:12.689346+0000 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.734975+0000 mon.a (mon.0) 202 : cluster [DBG] min_mon_release 19 (squid) 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.735063+0000 mon.a (mon.0) 203 : cluster [DBG] election_strategy: 1 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.735087+0000 mon.a (mon.0) 204 : cluster [DBG] 0: [v2:172.21.15.89:3300/0,v1:172.21.15.89:6789/0] mon.a 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.735109+0000 mon.a (mon.0) 205 : cluster [DBG] 1: [v2:172.21.15.203:3300/0,v1:172.21.15.203:6789/0] mon.b 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.735133+0000 mon.a (mon.0) 206 : cluster [DBG] 2: [v2:172.21.15.89:3301/0,v1:172.21.15.89:6790/0] mon.c 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.736539+0000 mon.a (mon.0) 207 : cluster [DBG] fsmap foo:1 {0=foo.smithi203.sjspdy=up:active} 1 up:standby 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.736597+0000 mon.a (mon.0) 208 : cluster [DBG] osdmap e92: 8 total, 8 up, 8 in 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.737320+0000 mon.a (mon.0) 209 : cluster [DBG] mgrmap e38: y(active, since 2m), standbys: x 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.737640+0000 mon.a (mon.0) 210 : cluster [INF] Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,c) 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761378+0000 mon.a (mon.0) 211 : cluster [ERR] Health detail: HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761445+0000 mon.a (mon.0) 212 : cluster [ERR] [ERR] OSD_UNREACHABLE: 8 osds(s) are not reachable 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761483+0000 mon.a (mon.0) 213 : cluster [ERR] osd.0's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761525+0000 mon.a (mon.0) 214 : cluster [ERR] osd.1's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.057 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761567+0000 mon.a (mon.0) 215 : cluster [ERR] osd.2's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761604+0000 mon.a (mon.0) 216 : cluster [ERR] osd.3's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761637+0000 mon.a (mon.0) 217 : cluster [ERR] osd.4's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761678+0000 mon.a (mon.0) 218 : cluster [ERR] osd.5's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761732+0000 mon.a (mon.0) 219 : cluster [ERR] osd.6's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:00.761773+0000 mon.a (mon.0) 220 : cluster [ERR] osd.7's public address is not in '172.21.0.2/32,172.21.0.0/20,172.21.15.254/32,172.21.0.1/32' subnet 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: audit 2024-04-13T07:40:00.770408+0000 mon.a (mon.0) 221 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: audit 2024-04-13T07:40:00.791780+0000 mon.a (mon.0) 222 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: audit 2024-04-13T07:40:00.805461+0000 mon.a (mon.0) 223 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:40:02.058 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:01 smithi089 bash[82331]: cluster 2024-04-13T07:40:01.008181+0000 mgr.y (mgr.34107) 87 : cluster [DBG] pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:02.610 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'sleep 60' 2024-04-13T07:40:02.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:02 smithi203 bash[60252]: ts=2024-04-13T07:40:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:03.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:02 smithi203 bash[68060]: audit 2024-04-13T07:40:01.869295+0000 mgr.y (mgr.34107) 88 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:40:03.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:02 smithi203 bash[68060]: cephadm 2024-04-13T07:40:01.869956+0000 mgr.y (mgr.34107) 89 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:40:03.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:02 smithi203 bash[68060]: audit 2024-04-13T07:40:01.878390+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:03.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:02 smithi089 bash[82331]: audit 2024-04-13T07:40:01.869295+0000 mgr.y (mgr.34107) 88 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:40:03.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:02 smithi089 bash[82331]: cephadm 2024-04-13T07:40:01.869956+0000 mgr.y (mgr.34107) 89 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:40:03.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:02 smithi089 bash[82331]: audit 2024-04-13T07:40:01.878390+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:03.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:02 smithi089 bash[84777]: audit 2024-04-13T07:40:01.869295+0000 mgr.y (mgr.34107) 88 : audit [DBG] from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:40:03.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:02 smithi089 bash[84777]: cephadm 2024-04-13T07:40:01.869956+0000 mgr.y (mgr.34107) 89 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:40:03.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:02 smithi089 bash[84777]: audit 2024-04-13T07:40:01.878390+0000 mon.a (mon.0) 224 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:03.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:03 smithi203 bash[60252]: ts=2024-04-13T07:40:03.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:04.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:03 smithi203 bash[68060]: cluster 2024-04-13T07:40:03.009245+0000 mgr.y (mgr.34107) 90 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:04.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:03 smithi089 bash[84777]: cluster 2024-04-13T07:40:03.009245+0000 mgr.y (mgr.34107) 90 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:04.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:03 smithi089 bash[82331]: cluster 2024-04-13T07:40:03.009245+0000 mgr.y (mgr.34107) 90 : cluster [DBG] pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:06.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:05 smithi203 bash[68060]: cluster 2024-04-13T07:40:05.009902+0000 mgr.y (mgr.34107) 91 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:06.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:40:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:40:05] "GET /metrics HTTP/1.1" 200 33875 "" "Prometheus/2.43.0" 2024-04-13T07:40:06.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:05 smithi089 bash[84777]: cluster 2024-04-13T07:40:05.009902+0000 mgr.y (mgr.34107) 91 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:06.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:05 smithi089 bash[82331]: cluster 2024-04-13T07:40:05.009902+0000 mgr.y (mgr.34107) 91 : cluster [DBG] pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:07.349 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:40:08.191 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:07 smithi089 bash[82331]: audit 2024-04-13T07:40:06.848534+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:08.191 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:07 smithi089 bash[82331]: audit 2024-04-13T07:40:06.856250+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:08.191 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:07 smithi089 bash[82331]: cluster 2024-04-13T07:40:07.010514+0000 mgr.y (mgr.34107) 92 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:08.191 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:07 smithi089 bash[84777]: audit 2024-04-13T07:40:06.848534+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:08.192 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:07 smithi089 bash[84777]: audit 2024-04-13T07:40:06.856250+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:08.192 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:07 smithi089 bash[84777]: cluster 2024-04-13T07:40:07.010514+0000 mgr.y (mgr.34107) 92 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:08.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:07 smithi203 bash[68060]: audit 2024-04-13T07:40:06.848534+0000 mon.a (mon.0) 225 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:08.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:07 smithi203 bash[68060]: audit 2024-04-13T07:40:06.856250+0000 mon.a (mon.0) 226 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:08.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:07 smithi203 bash[68060]: cluster 2024-04-13T07:40:07.010514+0000 mgr.y (mgr.34107) 92 : cluster [DBG] pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:09.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:08 smithi203 bash[68060]: audit 2024-04-13T07:40:07.901511+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:09.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:08 smithi203 bash[68060]: audit 2024-04-13T07:40:07.911837+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:09.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:08 smithi089 bash[84777]: audit 2024-04-13T07:40:07.901511+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:09.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:08 smithi089 bash[84777]: audit 2024-04-13T07:40:07.911837+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:09.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:08 smithi089 bash[82331]: audit 2024-04-13T07:40:07.901511+0000 mon.a (mon.0) 227 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:08 smithi089 bash[82331]: audit 2024-04-13T07:40:07.911837+0000 mon.a (mon.0) 228 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:10.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:09 smithi089 bash[82331]: cluster 2024-04-13T07:40:09.011495+0000 mgr.y (mgr.34107) 93 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:10.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:09 smithi089 bash[84777]: cluster 2024-04-13T07:40:09.011495+0000 mgr.y (mgr.34107) 93 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:10.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:09 smithi203 bash[68060]: cluster 2024-04-13T07:40:09.011495+0000 mgr.y (mgr.34107) 93 : cluster [DBG] pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:12.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:12 smithi203 bash[68060]: cluster 2024-04-13T07:40:11.012025+0000 mgr.y (mgr.34107) 94 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:12 smithi203 bash[68060]: audit 2024-04-13T07:40:12.061909+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:12.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:12 smithi203 bash[60252]: ts=2024-04-13T07:40:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:12 smithi089 bash[82331]: cluster 2024-04-13T07:40:11.012025+0000 mgr.y (mgr.34107) 94 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:12 smithi089 bash[82331]: audit 2024-04-13T07:40:12.061909+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:12.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:12 smithi089 bash[84777]: cluster 2024-04-13T07:40:11.012025+0000 mgr.y (mgr.34107) 94 : cluster [DBG] pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:12.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:12 smithi089 bash[84777]: audit 2024-04-13T07:40:12.061909+0000 mon.a (mon.0) 229 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:13.765 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:13 smithi203 bash[60252]: ts=2024-04-13T07:40:13.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:14.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:14 smithi203 bash[68060]: cluster 2024-04-13T07:40:13.013031+0000 mgr.y (mgr.34107) 95 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:14.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:14 smithi089 bash[82331]: cluster 2024-04-13T07:40:13.013031+0000 mgr.y (mgr.34107) 95 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:14.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:14 smithi089 bash[84777]: cluster 2024-04-13T07:40:13.013031+0000 mgr.y (mgr.34107) 95 : cluster [DBG] pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:16.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:40:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:40:15] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2024-04-13T07:40:16.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:16 smithi089 bash[84777]: cluster 2024-04-13T07:40:15.013662+0000 mgr.y (mgr.34107) 96 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:16.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:16 smithi089 bash[82331]: cluster 2024-04-13T07:40:15.013662+0000 mgr.y (mgr.34107) 96 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:16 smithi203 bash[68060]: cluster 2024-04-13T07:40:15.013662+0000 mgr.y (mgr.34107) 96 : cluster [DBG] pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: cluster 2024-04-13T07:40:17.014270+0000 mgr.y (mgr.34107) 97 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: audit 2024-04-13T07:40:17.921169+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: audit 2024-04-13T07:40:17.932716+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: audit 2024-04-13T07:40:17.935423+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: audit 2024-04-13T07:40:17.937378+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: audit 2024-04-13T07:40:17.947337+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: audit 2024-04-13T07:40:18.023059+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: audit 2024-04-13T07:40:18.024348+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:18.470 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[84777]: audit 2024-04-13T07:40:18.025504+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: cluster 2024-04-13T07:40:17.014270+0000 mgr.y (mgr.34107) 97 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: audit 2024-04-13T07:40:17.921169+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: audit 2024-04-13T07:40:17.932716+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: audit 2024-04-13T07:40:17.935423+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: audit 2024-04-13T07:40:17.937378+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: audit 2024-04-13T07:40:17.947337+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: audit 2024-04-13T07:40:18.023059+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: audit 2024-04-13T07:40:18.024348+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:18.471 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:18 smithi089 bash[82331]: audit 2024-04-13T07:40:18.025504+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:18.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: cluster 2024-04-13T07:40:17.014270+0000 mgr.y (mgr.34107) 97 : cluster [DBG] pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: audit 2024-04-13T07:40:17.921169+0000 mon.a (mon.0) 230 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: audit 2024-04-13T07:40:17.932716+0000 mon.a (mon.0) 231 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: audit 2024-04-13T07:40:17.935423+0000 mon.a (mon.0) 232 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: audit 2024-04-13T07:40:17.937378+0000 mon.a (mon.0) 233 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:40:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: audit 2024-04-13T07:40:17.947337+0000 mon.a (mon.0) 234 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: audit 2024-04-13T07:40:18.023059+0000 mon.a (mon.0) 235 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: audit 2024-04-13T07:40:18.024348+0000 mon.a (mon.0) 236 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:18 smithi203 bash[68060]: audit 2024-04-13T07:40:18.025504+0000 mon.a (mon.0) 237 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:19.361 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:19 smithi089 bash[82331]: cephadm 2024-04-13T07:40:18.022528+0000 mgr.y (mgr.34107) 98 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-04-13T07:40:19.361 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:19 smithi089 bash[82331]: cephadm 2024-04-13T07:40:18.027390+0000 mgr.y (mgr.34107) 99 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-04-13T07:40:19.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:19 smithi089 bash[84777]: cephadm 2024-04-13T07:40:18.022528+0000 mgr.y (mgr.34107) 98 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-04-13T07:40:19.362 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:19 smithi089 bash[84777]: cephadm 2024-04-13T07:40:18.027390+0000 mgr.y (mgr.34107) 99 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-04-13T07:40:19.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:19 smithi203 bash[68060]: cephadm 2024-04-13T07:40:18.022528+0000 mgr.y (mgr.34107) 98 : cephadm [INF] Reconfiguring mon.a (monmap changed)... 2024-04-13T07:40:19.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:19 smithi203 bash[68060]: cephadm 2024-04-13T07:40:18.027390+0000 mgr.y (mgr.34107) 99 : cephadm [INF] Reconfiguring daemon mon.a on smithi089 2024-04-13T07:40:20.433 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[84777]: cluster 2024-04-13T07:40:19.015328+0000 mgr.y (mgr.34107) 100 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[84777]: audit 2024-04-13T07:40:19.972105+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[84777]: audit 2024-04-13T07:40:19.985393+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[84777]: audit 2024-04-13T07:40:19.998129+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[84777]: audit 2024-04-13T07:40:19.999972+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[84777]: audit 2024-04-13T07:40:20.001940+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[82331]: cluster 2024-04-13T07:40:19.015328+0000 mgr.y (mgr.34107) 100 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[82331]: audit 2024-04-13T07:40:19.972105+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[82331]: audit 2024-04-13T07:40:19.985393+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:20.434 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[82331]: audit 2024-04-13T07:40:19.998129+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:40:20.435 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[82331]: audit 2024-04-13T07:40:19.999972+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:40:20.435 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:20 smithi089 bash[82331]: audit 2024-04-13T07:40:20.001940+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:20.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:20 smithi203 bash[68060]: cluster 2024-04-13T07:40:19.015328+0000 mgr.y (mgr.34107) 100 : cluster [DBG] pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:20.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:20 smithi203 bash[68060]: audit 2024-04-13T07:40:19.972105+0000 mon.a (mon.0) 238 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:20.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:20 smithi203 bash[68060]: audit 2024-04-13T07:40:19.985393+0000 mon.a (mon.0) 239 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:20.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:20 smithi203 bash[68060]: audit 2024-04-13T07:40:19.998129+0000 mon.a (mon.0) 240 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:40:20.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:20 smithi203 bash[68060]: audit 2024-04-13T07:40:19.999972+0000 mon.a (mon.0) 241 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:40:20.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:20 smithi203 bash[68060]: audit 2024-04-13T07:40:20.001940+0000 mon.a (mon.0) 242 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:21.413 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:21 smithi089 bash[84777]: cephadm 2024-04-13T07:40:19.997459+0000 mgr.y (mgr.34107) 101 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-04-13T07:40:21.413 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:21 smithi089 bash[84777]: cephadm 2024-04-13T07:40:20.003855+0000 mgr.y (mgr.34107) 102 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-04-13T07:40:21.413 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:21 smithi089 bash[82331]: cephadm 2024-04-13T07:40:19.997459+0000 mgr.y (mgr.34107) 101 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-04-13T07:40:21.413 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:21 smithi089 bash[82331]: cephadm 2024-04-13T07:40:20.003855+0000 mgr.y (mgr.34107) 102 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-04-13T07:40:21.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:21 smithi203 bash[68060]: cephadm 2024-04-13T07:40:19.997459+0000 mgr.y (mgr.34107) 101 : cephadm [INF] Reconfiguring mgr.y (monmap changed)... 2024-04-13T07:40:21.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:21 smithi203 bash[68060]: cephadm 2024-04-13T07:40:20.003855+0000 mgr.y (mgr.34107) 102 : cephadm [INF] Reconfiguring daemon mgr.y on smithi089 2024-04-13T07:40:22.514 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:22 smithi203 bash[60252]: ts=2024-04-13T07:40:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:22.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:22 smithi203 bash[68060]: cluster 2024-04-13T07:40:21.015942+0000 mgr.y (mgr.34107) 103 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:22.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:22 smithi203 bash[68060]: audit 2024-04-13T07:40:22.013948+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:22.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:22 smithi203 bash[68060]: audit 2024-04-13T07:40:22.023678+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:22.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:22 smithi203 bash[68060]: audit 2024-04-13T07:40:22.074542+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:22.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:22 smithi203 bash[68060]: audit 2024-04-13T07:40:22.076303+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:22.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:22 smithi203 bash[68060]: audit 2024-04-13T07:40:22.077906+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:22.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[82331]: cluster 2024-04-13T07:40:21.015942+0000 mgr.y (mgr.34107) 103 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:22.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[82331]: audit 2024-04-13T07:40:22.013948+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:22.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[82331]: audit 2024-04-13T07:40:22.023678+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:22.529 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[82331]: audit 2024-04-13T07:40:22.074542+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:22.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[82331]: audit 2024-04-13T07:40:22.076303+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:22.530 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[82331]: audit 2024-04-13T07:40:22.077906+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:22.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[84777]: cluster 2024-04-13T07:40:21.015942+0000 mgr.y (mgr.34107) 103 : cluster [DBG] pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:22.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[84777]: audit 2024-04-13T07:40:22.013948+0000 mon.a (mon.0) 243 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:22.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[84777]: audit 2024-04-13T07:40:22.023678+0000 mon.a (mon.0) 244 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:22.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[84777]: audit 2024-04-13T07:40:22.074542+0000 mon.a (mon.0) 245 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:22.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[84777]: audit 2024-04-13T07:40:22.076303+0000 mon.a (mon.0) 246 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:22.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:22 smithi089 bash[84777]: audit 2024-04-13T07:40:22.077906+0000 mon.a (mon.0) 247 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:23.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:23 smithi089 bash[84777]: cephadm 2024-04-13T07:40:22.073857+0000 mgr.y (mgr.34107) 104 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-04-13T07:40:23.409 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:23 smithi089 bash[84777]: cephadm 2024-04-13T07:40:22.079753+0000 mgr.y (mgr.34107) 105 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-04-13T07:40:23.410 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:23 smithi089 bash[82331]: cephadm 2024-04-13T07:40:22.073857+0000 mgr.y (mgr.34107) 104 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-04-13T07:40:23.410 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:23 smithi089 bash[82331]: cephadm 2024-04-13T07:40:22.079753+0000 mgr.y (mgr.34107) 105 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-04-13T07:40:23.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:23 smithi203 bash[68060]: cephadm 2024-04-13T07:40:22.073857+0000 mgr.y (mgr.34107) 104 : cephadm [INF] Reconfiguring mon.c (monmap changed)... 2024-04-13T07:40:23.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:23 smithi203 bash[68060]: cephadm 2024-04-13T07:40:22.079753+0000 mgr.y (mgr.34107) 105 : cephadm [INF] Reconfiguring daemon mon.c on smithi089 2024-04-13T07:40:23.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:23 smithi203 bash[60252]: ts=2024-04-13T07:40:23.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:24.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[84777]: cluster 2024-04-13T07:40:23.016906+0000 mgr.y (mgr.34107) 106 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:24.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[84777]: audit 2024-04-13T07:40:24.033328+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:24.500 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[84777]: audit 2024-04-13T07:40:24.044218+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:24.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[84777]: audit 2024-04-13T07:40:24.051396+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:40:24.501 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[84777]: audit 2024-04-13T07:40:24.053038+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:24.501 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[82331]: cluster 2024-04-13T07:40:23.016906+0000 mgr.y (mgr.34107) 106 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:24.501 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[82331]: audit 2024-04-13T07:40:24.033328+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:24.501 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[82331]: audit 2024-04-13T07:40:24.044218+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:24.501 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[82331]: audit 2024-04-13T07:40:24.051396+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:40:24.501 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:24 smithi089 bash[82331]: audit 2024-04-13T07:40:24.053038+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:24.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:24 smithi203 bash[68060]: cluster 2024-04-13T07:40:23.016906+0000 mgr.y (mgr.34107) 106 : cluster [DBG] pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:24.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:24 smithi203 bash[68060]: audit 2024-04-13T07:40:24.033328+0000 mon.a (mon.0) 248 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:24.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:24 smithi203 bash[68060]: audit 2024-04-13T07:40:24.044218+0000 mon.a (mon.0) 249 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:24.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:24 smithi203 bash[68060]: audit 2024-04-13T07:40:24.051396+0000 mon.a (mon.0) 250 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:40:24.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:24 smithi203 bash[68060]: audit 2024-04-13T07:40:24.053038+0000 mon.a (mon.0) 251 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:25.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:25 smithi089 bash[84777]: cephadm 2024-04-13T07:40:24.050741+0000 mgr.y (mgr.34107) 107 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-04-13T07:40:25.377 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:25 smithi089 bash[84777]: cephadm 2024-04-13T07:40:24.057962+0000 mgr.y (mgr.34107) 108 : cephadm [INF] Reconfiguring daemon osd.0 on smithi089 2024-04-13T07:40:25.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:25 smithi089 bash[82331]: cephadm 2024-04-13T07:40:24.050741+0000 mgr.y (mgr.34107) 107 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-04-13T07:40:25.377 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:25 smithi089 bash[82331]: cephadm 2024-04-13T07:40:24.057962+0000 mgr.y (mgr.34107) 108 : cephadm [INF] Reconfiguring daemon osd.0 on smithi089 2024-04-13T07:40:25.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:25 smithi203 bash[68060]: cephadm 2024-04-13T07:40:24.050741+0000 mgr.y (mgr.34107) 107 : cephadm [INF] Reconfiguring osd.0 (monmap changed)... 2024-04-13T07:40:25.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:25 smithi203 bash[68060]: cephadm 2024-04-13T07:40:24.057962+0000 mgr.y (mgr.34107) 108 : cephadm [INF] Reconfiguring daemon osd.0 on smithi089 2024-04-13T07:40:26.120 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:40:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:40:25] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2024-04-13T07:40:26.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[82331]: cluster 2024-04-13T07:40:25.017649+0000 mgr.y (mgr.34107) 109 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:26.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[82331]: audit 2024-04-13T07:40:26.028413+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:26.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[82331]: audit 2024-04-13T07:40:26.039022+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:26.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[82331]: audit 2024-04-13T07:40:26.078639+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:40:26.121 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[82331]: audit 2024-04-13T07:40:26.080395+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:26.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:26 smithi203 bash[68060]: cluster 2024-04-13T07:40:25.017649+0000 mgr.y (mgr.34107) 109 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:26.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:26 smithi203 bash[68060]: audit 2024-04-13T07:40:26.028413+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:26.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:26 smithi203 bash[68060]: audit 2024-04-13T07:40:26.039022+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:26.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:26 smithi203 bash[68060]: audit 2024-04-13T07:40:26.078639+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:40:26.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:26 smithi203 bash[68060]: audit 2024-04-13T07:40:26.080395+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:26.521 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[84777]: cluster 2024-04-13T07:40:25.017649+0000 mgr.y (mgr.34107) 109 : cluster [DBG] pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:26.521 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[84777]: audit 2024-04-13T07:40:26.028413+0000 mon.a (mon.0) 252 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:26.522 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[84777]: audit 2024-04-13T07:40:26.039022+0000 mon.a (mon.0) 253 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:26.522 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[84777]: audit 2024-04-13T07:40:26.078639+0000 mon.a (mon.0) 254 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:40:26.522 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:26 smithi089 bash[84777]: audit 2024-04-13T07:40:26.080395+0000 mon.a (mon.0) 255 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:27.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:27 smithi089 bash[82331]: cephadm 2024-04-13T07:40:26.077964+0000 mgr.y (mgr.34107) 110 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-04-13T07:40:27.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:27 smithi089 bash[82331]: cephadm 2024-04-13T07:40:26.085172+0000 mgr.y (mgr.34107) 111 : cephadm [INF] Reconfiguring daemon osd.1 on smithi089 2024-04-13T07:40:27.390 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:27 smithi089 bash[82331]: audit 2024-04-13T07:40:27.062232+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:27.390 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:27 smithi089 bash[84777]: cephadm 2024-04-13T07:40:26.077964+0000 mgr.y (mgr.34107) 110 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-04-13T07:40:27.390 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:27 smithi089 bash[84777]: cephadm 2024-04-13T07:40:26.085172+0000 mgr.y (mgr.34107) 111 : cephadm [INF] Reconfiguring daemon osd.1 on smithi089 2024-04-13T07:40:27.390 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:27 smithi089 bash[84777]: audit 2024-04-13T07:40:27.062232+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:27.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:27 smithi203 bash[68060]: cephadm 2024-04-13T07:40:26.077964+0000 mgr.y (mgr.34107) 110 : cephadm [INF] Reconfiguring osd.1 (monmap changed)... 2024-04-13T07:40:27.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:27 smithi203 bash[68060]: cephadm 2024-04-13T07:40:26.085172+0000 mgr.y (mgr.34107) 111 : cephadm [INF] Reconfiguring daemon osd.1 on smithi089 2024-04-13T07:40:27.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:27 smithi203 bash[68060]: audit 2024-04-13T07:40:27.062232+0000 mon.a (mon.0) 256 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[84777]: cluster 2024-04-13T07:40:27.018363+0000 mgr.y (mgr.34107) 112 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[84777]: audit 2024-04-13T07:40:28.013355+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[84777]: audit 2024-04-13T07:40:28.022074+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[84777]: audit 2024-04-13T07:40:28.060332+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[84777]: audit 2024-04-13T07:40:28.062037+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[82331]: cluster 2024-04-13T07:40:27.018363+0000 mgr.y (mgr.34107) 112 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[82331]: audit 2024-04-13T07:40:28.013355+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[82331]: audit 2024-04-13T07:40:28.022074+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:28.506 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[82331]: audit 2024-04-13T07:40:28.060332+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:40:28.507 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:28 smithi089 bash[82331]: audit 2024-04-13T07:40:28.062037+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:28.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:28 smithi203 bash[68060]: cluster 2024-04-13T07:40:27.018363+0000 mgr.y (mgr.34107) 112 : cluster [DBG] pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:28.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:28 smithi203 bash[68060]: audit 2024-04-13T07:40:28.013355+0000 mon.a (mon.0) 257 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:28.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:28 smithi203 bash[68060]: audit 2024-04-13T07:40:28.022074+0000 mon.a (mon.0) 258 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:28.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:28 smithi203 bash[68060]: audit 2024-04-13T07:40:28.060332+0000 mon.a (mon.0) 259 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:40:28.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:28 smithi203 bash[68060]: audit 2024-04-13T07:40:28.062037+0000 mon.a (mon.0) 260 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:29.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:29 smithi203 bash[68060]: cephadm 2024-04-13T07:40:28.059722+0000 mgr.y (mgr.34107) 113 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-04-13T07:40:29.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:29 smithi203 bash[68060]: cephadm 2024-04-13T07:40:28.066579+0000 mgr.y (mgr.34107) 114 : cephadm [INF] Reconfiguring daemon osd.2 on smithi089 2024-04-13T07:40:29.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:29 smithi089 bash[84777]: cephadm 2024-04-13T07:40:28.059722+0000 mgr.y (mgr.34107) 113 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-04-13T07:40:29.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:29 smithi089 bash[84777]: cephadm 2024-04-13T07:40:28.066579+0000 mgr.y (mgr.34107) 114 : cephadm [INF] Reconfiguring daemon osd.2 on smithi089 2024-04-13T07:40:29.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:29 smithi089 bash[82331]: cephadm 2024-04-13T07:40:28.059722+0000 mgr.y (mgr.34107) 113 : cephadm [INF] Reconfiguring osd.2 (monmap changed)... 2024-04-13T07:40:29.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:29 smithi089 bash[82331]: cephadm 2024-04-13T07:40:28.066579+0000 mgr.y (mgr.34107) 114 : cephadm [INF] Reconfiguring daemon osd.2 on smithi089 2024-04-13T07:40:30.479 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[84777]: cluster 2024-04-13T07:40:29.019564+0000 mgr.y (mgr.34107) 115 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:30.479 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[84777]: audit 2024-04-13T07:40:29.990524+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:30.480 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[84777]: audit 2024-04-13T07:40:30.001638+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:30.480 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[84777]: audit 2024-04-13T07:40:30.037872+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:40:30.480 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[84777]: audit 2024-04-13T07:40:30.039355+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[82331]: cluster 2024-04-13T07:40:29.019564+0000 mgr.y (mgr.34107) 115 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[82331]: audit 2024-04-13T07:40:29.990524+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[82331]: audit 2024-04-13T07:40:30.001638+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[82331]: audit 2024-04-13T07:40:30.037872+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:40:30.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:30 smithi089 bash[82331]: audit 2024-04-13T07:40:30.039355+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:30.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:30 smithi203 bash[68060]: cluster 2024-04-13T07:40:29.019564+0000 mgr.y (mgr.34107) 115 : cluster [DBG] pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:30.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:30 smithi203 bash[68060]: audit 2024-04-13T07:40:29.990524+0000 mon.a (mon.0) 261 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:30.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:30 smithi203 bash[68060]: audit 2024-04-13T07:40:30.001638+0000 mon.a (mon.0) 262 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:30.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:30 smithi203 bash[68060]: audit 2024-04-13T07:40:30.037872+0000 mon.a (mon.0) 263 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:40:30.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:30 smithi203 bash[68060]: audit 2024-04-13T07:40:30.039355+0000 mon.a (mon.0) 264 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:31.480 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:31 smithi089 bash[84777]: cephadm 2024-04-13T07:40:30.037223+0000 mgr.y (mgr.34107) 116 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-04-13T07:40:31.480 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:31 smithi089 bash[84777]: cephadm 2024-04-13T07:40:30.043559+0000 mgr.y (mgr.34107) 117 : cephadm [INF] Reconfiguring daemon osd.3 on smithi089 2024-04-13T07:40:31.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:31 smithi089 bash[82331]: cephadm 2024-04-13T07:40:30.037223+0000 mgr.y (mgr.34107) 116 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-04-13T07:40:31.480 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:31 smithi089 bash[82331]: cephadm 2024-04-13T07:40:30.043559+0000 mgr.y (mgr.34107) 117 : cephadm [INF] Reconfiguring daemon osd.3 on smithi089 2024-04-13T07:40:31.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:31 smithi203 bash[68060]: cephadm 2024-04-13T07:40:30.037223+0000 mgr.y (mgr.34107) 116 : cephadm [INF] Reconfiguring osd.3 (monmap changed)... 2024-04-13T07:40:31.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:31 smithi203 bash[68060]: cephadm 2024-04-13T07:40:30.043559+0000 mgr.y (mgr.34107) 117 : cephadm [INF] Reconfiguring daemon osd.3 on smithi089 2024-04-13T07:40:32.424 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:32 smithi203 bash[68060]: cluster 2024-04-13T07:40:31.020258+0000 mgr.y (mgr.34107) 118 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:32.425 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:32 smithi203 bash[68060]: audit 2024-04-13T07:40:32.001605+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:32.425 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:32 smithi203 bash[68060]: audit 2024-04-13T07:40:32.011536+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:32.425 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:32 smithi203 bash[68060]: audit 2024-04-13T07:40:32.063800+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:40:32.425 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:32 smithi203 bash[68060]: audit 2024-04-13T07:40:32.065989+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:32.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[84777]: cluster 2024-04-13T07:40:31.020258+0000 mgr.y (mgr.34107) 118 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:32.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[84777]: audit 2024-04-13T07:40:32.001605+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:32.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[84777]: audit 2024-04-13T07:40:32.011536+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:32.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[84777]: audit 2024-04-13T07:40:32.063800+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:40:32.508 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[84777]: audit 2024-04-13T07:40:32.065989+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:32.509 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[82331]: cluster 2024-04-13T07:40:31.020258+0000 mgr.y (mgr.34107) 118 : cluster [DBG] pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:32.509 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[82331]: audit 2024-04-13T07:40:32.001605+0000 mon.a (mon.0) 265 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:32.509 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[82331]: audit 2024-04-13T07:40:32.011536+0000 mon.a (mon.0) 266 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:32.509 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[82331]: audit 2024-04-13T07:40:32.063800+0000 mon.a (mon.0) 267 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi089.pccnvq", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:40:32.509 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:32 smithi089 bash[82331]: audit 2024-04-13T07:40:32.065989+0000 mon.a (mon.0) 268 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:32.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:32 smithi203 bash[60252]: ts=2024-04-13T07:40:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:33.433 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:33 smithi089 bash[84777]: cephadm 2024-04-13T07:40:32.062962+0000 mgr.y (mgr.34107) 119 : cephadm [INF] Reconfiguring mds.foo.smithi089.pccnvq (monmap changed)... 2024-04-13T07:40:33.434 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:33 smithi089 bash[84777]: cephadm 2024-04-13T07:40:32.067995+0000 mgr.y (mgr.34107) 120 : cephadm [INF] Reconfiguring daemon mds.foo.smithi089.pccnvq on smithi089 2024-04-13T07:40:33.434 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:33 smithi089 bash[82331]: cephadm 2024-04-13T07:40:32.062962+0000 mgr.y (mgr.34107) 119 : cephadm [INF] Reconfiguring mds.foo.smithi089.pccnvq (monmap changed)... 2024-04-13T07:40:33.434 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:33 smithi089 bash[82331]: cephadm 2024-04-13T07:40:32.067995+0000 mgr.y (mgr.34107) 120 : cephadm [INF] Reconfiguring daemon mds.foo.smithi089.pccnvq on smithi089 2024-04-13T07:40:33.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:33 smithi203 bash[68060]: cephadm 2024-04-13T07:40:32.062962+0000 mgr.y (mgr.34107) 119 : cephadm [INF] Reconfiguring mds.foo.smithi089.pccnvq (monmap changed)... 2024-04-13T07:40:33.502 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:33 smithi203 bash[68060]: cephadm 2024-04-13T07:40:32.067995+0000 mgr.y (mgr.34107) 120 : cephadm [INF] Reconfiguring daemon mds.foo.smithi089.pccnvq on smithi089 2024-04-13T07:40:33.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:33 smithi203 bash[60252]: ts=2024-04-13T07:40:33.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:34.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:34 smithi203 bash[68060]: cluster 2024-04-13T07:40:33.021298+0000 mgr.y (mgr.34107) 121 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:34.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:34 smithi203 bash[68060]: audit 2024-04-13T07:40:34.078692+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:34.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:34 smithi203 bash[68060]: audit 2024-04-13T07:40:34.087595+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:34.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:34 smithi203 bash[68060]: audit 2024-04-13T07:40:34.138474+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:34.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:34 smithi203 bash[68060]: audit 2024-04-13T07:40:34.140273+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:34.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:34 smithi203 bash[68060]: audit 2024-04-13T07:40:34.141967+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:34.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[82331]: cluster 2024-04-13T07:40:33.021298+0000 mgr.y (mgr.34107) 121 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:34.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[82331]: audit 2024-04-13T07:40:34.078692+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:34.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[82331]: audit 2024-04-13T07:40:34.087595+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:34.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[82331]: audit 2024-04-13T07:40:34.138474+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:34.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[82331]: audit 2024-04-13T07:40:34.140273+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:34.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[82331]: audit 2024-04-13T07:40:34.141967+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:34.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[84777]: cluster 2024-04-13T07:40:33.021298+0000 mgr.y (mgr.34107) 121 : cluster [DBG] pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:34.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[84777]: audit 2024-04-13T07:40:34.078692+0000 mon.a (mon.0) 269 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:34.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[84777]: audit 2024-04-13T07:40:34.087595+0000 mon.a (mon.0) 270 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:34.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[84777]: audit 2024-04-13T07:40:34.138474+0000 mon.a (mon.0) 271 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2024-04-13T07:40:34.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[84777]: audit 2024-04-13T07:40:34.140273+0000 mon.a (mon.0) 272 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2024-04-13T07:40:34.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:34 smithi089 bash[84777]: audit 2024-04-13T07:40:34.141967+0000 mon.a (mon.0) 273 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:35.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:35 smithi203 bash[68060]: cephadm 2024-04-13T07:40:34.137806+0000 mgr.y (mgr.34107) 122 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-04-13T07:40:35.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:35 smithi203 bash[68060]: cephadm 2024-04-13T07:40:34.144199+0000 mgr.y (mgr.34107) 123 : cephadm [INF] Reconfiguring daemon mon.b on smithi203 2024-04-13T07:40:35.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:35 smithi089 bash[82331]: cephadm 2024-04-13T07:40:34.137806+0000 mgr.y (mgr.34107) 122 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-04-13T07:40:35.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:35 smithi089 bash[82331]: cephadm 2024-04-13T07:40:34.144199+0000 mgr.y (mgr.34107) 123 : cephadm [INF] Reconfiguring daemon mon.b on smithi203 2024-04-13T07:40:35.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:35 smithi089 bash[84777]: cephadm 2024-04-13T07:40:34.137806+0000 mgr.y (mgr.34107) 122 : cephadm [INF] Reconfiguring mon.b (monmap changed)... 2024-04-13T07:40:35.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:35 smithi089 bash[84777]: cephadm 2024-04-13T07:40:34.144199+0000 mgr.y (mgr.34107) 123 : cephadm [INF] Reconfiguring daemon mon.b on smithi203 2024-04-13T07:40:36.184 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:40:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:40:35] "GET /metrics HTTP/1.1" 200 33890 "" "Prometheus/2.43.0" 2024-04-13T07:40:36.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:36 smithi203 bash[68060]: cluster 2024-04-13T07:40:35.021905+0000 mgr.y (mgr.34107) 124 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:36.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:36 smithi089 bash[82331]: cluster 2024-04-13T07:40:35.021905+0000 mgr.y (mgr.34107) 124 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:36.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:36 smithi089 bash[84777]: cluster 2024-04-13T07:40:35.021905+0000 mgr.y (mgr.34107) 124 : cluster [DBG] pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:37.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:37 smithi203 bash[68060]: audit 2024-04-13T07:40:36.487875+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:37.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:37 smithi203 bash[68060]: audit 2024-04-13T07:40:36.499301+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:37.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:37 smithi203 bash[68060]: cephadm 2024-04-13T07:40:36.515689+0000 mgr.y (mgr.34107) 125 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-04-13T07:40:37.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:37 smithi203 bash[68060]: audit 2024-04-13T07:40:36.516484+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:40:37.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:37 smithi203 bash[68060]: audit 2024-04-13T07:40:36.518539+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:40:37.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:37 smithi203 bash[68060]: audit 2024-04-13T07:40:36.520170+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:37.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:37 smithi203 bash[68060]: cephadm 2024-04-13T07:40:36.522047+0000 mgr.y (mgr.34107) 126 : cephadm [INF] Reconfiguring daemon mgr.x on smithi203 2024-04-13T07:40:37.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[82331]: audit 2024-04-13T07:40:36.487875+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:37.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[82331]: audit 2024-04-13T07:40:36.499301+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:37.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[82331]: cephadm 2024-04-13T07:40:36.515689+0000 mgr.y (mgr.34107) 125 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-04-13T07:40:37.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[82331]: audit 2024-04-13T07:40:36.516484+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[82331]: audit 2024-04-13T07:40:36.518539+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[82331]: audit 2024-04-13T07:40:36.520170+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[82331]: cephadm 2024-04-13T07:40:36.522047+0000 mgr.y (mgr.34107) 126 : cephadm [INF] Reconfiguring daemon mgr.x on smithi203 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[84777]: audit 2024-04-13T07:40:36.487875+0000 mon.a (mon.0) 274 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[84777]: audit 2024-04-13T07:40:36.499301+0000 mon.a (mon.0) 275 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[84777]: cephadm 2024-04-13T07:40:36.515689+0000 mgr.y (mgr.34107) 125 : cephadm [INF] Reconfiguring mgr.x (monmap changed)... 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[84777]: audit 2024-04-13T07:40:36.516484+0000 mon.a (mon.0) 276 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[84777]: audit 2024-04-13T07:40:36.518539+0000 mon.a (mon.0) 277 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[84777]: audit 2024-04-13T07:40:36.520170+0000 mon.a (mon.0) 278 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:37.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:37 smithi089 bash[84777]: cephadm 2024-04-13T07:40:36.522047+0000 mgr.y (mgr.34107) 126 : cephadm [INF] Reconfiguring daemon mgr.x on smithi203 2024-04-13T07:40:38.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:38 smithi203 bash[68060]: cluster 2024-04-13T07:40:37.022510+0000 mgr.y (mgr.34107) 127 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:38.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:38 smithi089 bash[82331]: cluster 2024-04-13T07:40:37.022510+0000 mgr.y (mgr.34107) 127 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:38.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:38 smithi089 bash[84777]: cluster 2024-04-13T07:40:37.022510+0000 mgr.y (mgr.34107) 127 : cluster [DBG] pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:40.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:39 smithi203 bash[68060]: audit 2024-04-13T07:40:38.733260+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:40.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:39 smithi203 bash[68060]: audit 2024-04-13T07:40:38.744929+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:40.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:39 smithi203 bash[68060]: audit 2024-04-13T07:40:38.795295+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:40:40.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:39 smithi203 bash[68060]: audit 2024-04-13T07:40:38.797089+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:40.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:39 smithi089 bash[82331]: audit 2024-04-13T07:40:38.733260+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:40.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:39 smithi089 bash[82331]: audit 2024-04-13T07:40:38.744929+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:40.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:39 smithi089 bash[82331]: audit 2024-04-13T07:40:38.795295+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:40:40.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:39 smithi089 bash[82331]: audit 2024-04-13T07:40:38.797089+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:40.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:39 smithi089 bash[84777]: audit 2024-04-13T07:40:38.733260+0000 mon.a (mon.0) 279 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:40.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:39 smithi089 bash[84777]: audit 2024-04-13T07:40:38.744929+0000 mon.a (mon.0) 280 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:40.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:39 smithi089 bash[84777]: audit 2024-04-13T07:40:38.795295+0000 mon.a (mon.0) 281 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:40:40.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:39 smithi089 bash[84777]: audit 2024-04-13T07:40:38.797089+0000 mon.a (mon.0) 282 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:40.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:40 smithi203 bash[68060]: cephadm 2024-04-13T07:40:38.794612+0000 mgr.y (mgr.34107) 128 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-04-13T07:40:40.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:40 smithi203 bash[68060]: cephadm 2024-04-13T07:40:38.801890+0000 mgr.y (mgr.34107) 129 : cephadm [INF] Reconfiguring daemon osd.4 on smithi203 2024-04-13T07:40:40.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:40 smithi203 bash[68060]: cluster 2024-04-13T07:40:39.023658+0000 mgr.y (mgr.34107) 130 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:41.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:40 smithi089 bash[84777]: cephadm 2024-04-13T07:40:38.794612+0000 mgr.y (mgr.34107) 128 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-04-13T07:40:41.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:40 smithi089 bash[84777]: cephadm 2024-04-13T07:40:38.801890+0000 mgr.y (mgr.34107) 129 : cephadm [INF] Reconfiguring daemon osd.4 on smithi203 2024-04-13T07:40:41.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:40 smithi089 bash[84777]: cluster 2024-04-13T07:40:39.023658+0000 mgr.y (mgr.34107) 130 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:41.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:40 smithi089 bash[82331]: cephadm 2024-04-13T07:40:38.794612+0000 mgr.y (mgr.34107) 128 : cephadm [INF] Reconfiguring osd.4 (monmap changed)... 2024-04-13T07:40:41.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:40 smithi089 bash[82331]: cephadm 2024-04-13T07:40:38.801890+0000 mgr.y (mgr.34107) 129 : cephadm [INF] Reconfiguring daemon osd.4 on smithi203 2024-04-13T07:40:41.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:40 smithi089 bash[82331]: cluster 2024-04-13T07:40:39.023658+0000 mgr.y (mgr.34107) 130 : cluster [DBG] pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:42.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:41 smithi203 bash[68060]: audit 2024-04-13T07:40:40.994522+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:42.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:41 smithi203 bash[68060]: audit 2024-04-13T07:40:41.006128+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:42.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:41 smithi203 bash[68060]: cephadm 2024-04-13T07:40:41.023011+0000 mgr.y (mgr.34107) 131 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-04-13T07:40:42.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:41 smithi203 bash[68060]: audit 2024-04-13T07:40:41.023741+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-13T07:40:42.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:41 smithi203 bash[68060]: cluster 2024-04-13T07:40:41.024218+0000 mgr.y (mgr.34107) 132 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:42.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:41 smithi203 bash[68060]: audit 2024-04-13T07:40:41.026845+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:42.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:41 smithi203 bash[68060]: cephadm 2024-04-13T07:40:41.031886+0000 mgr.y (mgr.34107) 133 : cephadm [INF] Reconfiguring daemon osd.5 on smithi203 2024-04-13T07:40:42.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[82331]: audit 2024-04-13T07:40:40.994522+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[82331]: audit 2024-04-13T07:40:41.006128+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[82331]: cephadm 2024-04-13T07:40:41.023011+0000 mgr.y (mgr.34107) 131 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-04-13T07:40:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[82331]: audit 2024-04-13T07:40:41.023741+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-13T07:40:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[82331]: cluster 2024-04-13T07:40:41.024218+0000 mgr.y (mgr.34107) 132 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[82331]: audit 2024-04-13T07:40:41.026845+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:42.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[82331]: cephadm 2024-04-13T07:40:41.031886+0000 mgr.y (mgr.34107) 133 : cephadm [INF] Reconfiguring daemon osd.5 on smithi203 2024-04-13T07:40:42.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[84777]: audit 2024-04-13T07:40:40.994522+0000 mon.a (mon.0) 283 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:42.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[84777]: audit 2024-04-13T07:40:41.006128+0000 mon.a (mon.0) 284 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:42.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[84777]: cephadm 2024-04-13T07:40:41.023011+0000 mgr.y (mgr.34107) 131 : cephadm [INF] Reconfiguring osd.5 (monmap changed)... 2024-04-13T07:40:42.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[84777]: audit 2024-04-13T07:40:41.023741+0000 mon.a (mon.0) 285 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2024-04-13T07:40:42.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[84777]: cluster 2024-04-13T07:40:41.024218+0000 mgr.y (mgr.34107) 132 : cluster [DBG] pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:42.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[84777]: audit 2024-04-13T07:40:41.026845+0000 mon.a (mon.0) 286 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:42.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:41 smithi089 bash[84777]: cephadm 2024-04-13T07:40:41.031886+0000 mgr.y (mgr.34107) 133 : cephadm [INF] Reconfiguring daemon osd.5 on smithi203 2024-04-13T07:40:42.640 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:42 smithi203 bash[60252]: ts=2024-04-13T07:40:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:43.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:43 smithi203 bash[68060]: audit 2024-04-13T07:40:42.062938+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:43.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:43 smithi089 bash[84777]: audit 2024-04-13T07:40:42.062938+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:43.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:43 smithi089 bash[82331]: audit 2024-04-13T07:40:42.062938+0000 mon.a (mon.0) 287 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:43.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:43 smithi203 bash[60252]: ts=2024-04-13T07:40:43.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:44.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:44 smithi203 bash[68060]: cluster 2024-04-13T07:40:43.025245+0000 mgr.y (mgr.34107) 134 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:44.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:44 smithi203 bash[68060]: audit 2024-04-13T07:40:43.441231+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:44.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:44 smithi203 bash[68060]: audit 2024-04-13T07:40:43.452658+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:44.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:44 smithi203 bash[68060]: cephadm 2024-04-13T07:40:43.456415+0000 mgr.y (mgr.34107) 135 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-04-13T07:40:44.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:44 smithi203 bash[68060]: audit 2024-04-13T07:40:43.457122+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-13T07:40:44.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:44 smithi203 bash[68060]: audit 2024-04-13T07:40:43.459074+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:44.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:44 smithi203 bash[68060]: cephadm 2024-04-13T07:40:43.463876+0000 mgr.y (mgr.34107) 136 : cephadm [INF] Reconfiguring daemon osd.6 on smithi203 2024-04-13T07:40:44.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[84777]: cluster 2024-04-13T07:40:43.025245+0000 mgr.y (mgr.34107) 134 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:44.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[84777]: audit 2024-04-13T07:40:43.441231+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:44.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[84777]: audit 2024-04-13T07:40:43.452658+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:44.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[84777]: cephadm 2024-04-13T07:40:43.456415+0000 mgr.y (mgr.34107) 135 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[84777]: audit 2024-04-13T07:40:43.457122+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[84777]: audit 2024-04-13T07:40:43.459074+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[84777]: cephadm 2024-04-13T07:40:43.463876+0000 mgr.y (mgr.34107) 136 : cephadm [INF] Reconfiguring daemon osd.6 on smithi203 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[82331]: cluster 2024-04-13T07:40:43.025245+0000 mgr.y (mgr.34107) 134 : cluster [DBG] pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[82331]: audit 2024-04-13T07:40:43.441231+0000 mon.a (mon.0) 288 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[82331]: audit 2024-04-13T07:40:43.452658+0000 mon.a (mon.0) 289 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[82331]: cephadm 2024-04-13T07:40:43.456415+0000 mgr.y (mgr.34107) 135 : cephadm [INF] Reconfiguring osd.6 (monmap changed)... 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[82331]: audit 2024-04-13T07:40:43.457122+0000 mon.a (mon.0) 290 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[82331]: audit 2024-04-13T07:40:43.459074+0000 mon.a (mon.0) 291 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:44.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:44 smithi089 bash[82331]: cephadm 2024-04-13T07:40:43.463876+0000 mgr.y (mgr.34107) 136 : cephadm [INF] Reconfiguring daemon osd.6 on smithi203 2024-04-13T07:40:46.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:40:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:40:45] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.43.0" 2024-04-13T07:40:46.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:46 smithi203 bash[68060]: cluster 2024-04-13T07:40:45.025907+0000 mgr.y (mgr.34107) 137 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:46.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:46 smithi203 bash[68060]: audit 2024-04-13T07:40:45.831047+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:46.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:46 smithi203 bash[68060]: audit 2024-04-13T07:40:45.842647+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:46.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:46 smithi203 bash[68060]: audit 2024-04-13T07:40:45.886077+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-13T07:40:46.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:46 smithi203 bash[68060]: audit 2024-04-13T07:40:45.888043+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:46.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[84777]: cluster 2024-04-13T07:40:45.025907+0000 mgr.y (mgr.34107) 137 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:46.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[84777]: audit 2024-04-13T07:40:45.831047+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:46.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[84777]: audit 2024-04-13T07:40:45.842647+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:46.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[84777]: audit 2024-04-13T07:40:45.886077+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-13T07:40:46.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[84777]: audit 2024-04-13T07:40:45.888043+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:46.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[82331]: cluster 2024-04-13T07:40:45.025907+0000 mgr.y (mgr.34107) 137 : cluster [DBG] pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:46.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[82331]: audit 2024-04-13T07:40:45.831047+0000 mon.a (mon.0) 292 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:46.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[82331]: audit 2024-04-13T07:40:45.842647+0000 mon.a (mon.0) 293 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:46.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[82331]: audit 2024-04-13T07:40:45.886077+0000 mon.a (mon.0) 294 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2024-04-13T07:40:46.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:46 smithi089 bash[82331]: audit 2024-04-13T07:40:45.888043+0000 mon.a (mon.0) 295 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:47.696 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:47 smithi203 bash[68060]: cephadm 2024-04-13T07:40:45.885399+0000 mgr.y (mgr.34107) 138 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-04-13T07:40:47.697 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:47 smithi203 bash[68060]: cephadm 2024-04-13T07:40:45.892880+0000 mgr.y (mgr.34107) 139 : cephadm [INF] Reconfiguring daemon osd.7 on smithi203 2024-04-13T07:40:47.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:47 smithi089 bash[84777]: cephadm 2024-04-13T07:40:45.885399+0000 mgr.y (mgr.34107) 138 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-04-13T07:40:47.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:47 smithi089 bash[84777]: cephadm 2024-04-13T07:40:45.892880+0000 mgr.y (mgr.34107) 139 : cephadm [INF] Reconfiguring daemon osd.7 on smithi203 2024-04-13T07:40:47.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:47 smithi089 bash[82331]: cephadm 2024-04-13T07:40:45.885399+0000 mgr.y (mgr.34107) 138 : cephadm [INF] Reconfiguring osd.7 (monmap changed)... 2024-04-13T07:40:47.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:47 smithi089 bash[82331]: cephadm 2024-04-13T07:40:45.892880+0000 mgr.y (mgr.34107) 139 : cephadm [INF] Reconfiguring daemon osd.7 on smithi203 2024-04-13T07:40:48.713 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:48 smithi203 bash[68060]: cluster 2024-04-13T07:40:47.026559+0000 mgr.y (mgr.34107) 140 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:48.713 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:48 smithi203 bash[68060]: audit 2024-04-13T07:40:48.214877+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:48.713 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:48 smithi203 bash[68060]: audit 2024-04-13T07:40:48.228194+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:48.713 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:48 smithi203 bash[68060]: audit 2024-04-13T07:40:48.265007+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:40:48.713 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:48 smithi203 bash[68060]: audit 2024-04-13T07:40:48.267060+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:48.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[84777]: cluster 2024-04-13T07:40:47.026559+0000 mgr.y (mgr.34107) 140 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[84777]: audit 2024-04-13T07:40:48.214877+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[84777]: audit 2024-04-13T07:40:48.228194+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[84777]: audit 2024-04-13T07:40:48.265007+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[84777]: audit 2024-04-13T07:40:48.267060+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[82331]: cluster 2024-04-13T07:40:47.026559+0000 mgr.y (mgr.34107) 140 : cluster [DBG] pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[82331]: audit 2024-04-13T07:40:48.214877+0000 mon.a (mon.0) 296 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[82331]: audit 2024-04-13T07:40:48.228194+0000 mon.a (mon.0) 297 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[82331]: audit 2024-04-13T07:40:48.265007+0000 mon.a (mon.0) 298 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi203.sjspdy", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2024-04-13T07:40:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:48 smithi089 bash[82331]: audit 2024-04-13T07:40:48.267060+0000 mon.a (mon.0) 299 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:49.753 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:49 smithi203 bash[68060]: cephadm 2024-04-13T07:40:48.264317+0000 mgr.y (mgr.34107) 141 : cephadm [INF] Reconfiguring mds.foo.smithi203.sjspdy (monmap changed)... 2024-04-13T07:40:49.753 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:49 smithi203 bash[68060]: cephadm 2024-04-13T07:40:48.269059+0000 mgr.y (mgr.34107) 142 : cephadm [INF] Reconfiguring daemon mds.foo.smithi203.sjspdy on smithi203 2024-04-13T07:40:49.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:49 smithi089 bash[84777]: cephadm 2024-04-13T07:40:48.264317+0000 mgr.y (mgr.34107) 141 : cephadm [INF] Reconfiguring mds.foo.smithi203.sjspdy (monmap changed)... 2024-04-13T07:40:49.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:49 smithi089 bash[84777]: cephadm 2024-04-13T07:40:48.269059+0000 mgr.y (mgr.34107) 142 : cephadm [INF] Reconfiguring daemon mds.foo.smithi203.sjspdy on smithi203 2024-04-13T07:40:49.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:49 smithi089 bash[82331]: cephadm 2024-04-13T07:40:48.264317+0000 mgr.y (mgr.34107) 141 : cephadm [INF] Reconfiguring mds.foo.smithi203.sjspdy (monmap changed)... 2024-04-13T07:40:49.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:49 smithi089 bash[82331]: cephadm 2024-04-13T07:40:48.269059+0000 mgr.y (mgr.34107) 142 : cephadm [INF] Reconfiguring daemon mds.foo.smithi203.sjspdy on smithi203 2024-04-13T07:40:50.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:50 smithi203 bash[68060]: cluster 2024-04-13T07:40:49.027731+0000 mgr.y (mgr.34107) 143 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:50.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:50 smithi089 bash[84777]: cluster 2024-04-13T07:40:49.027731+0000 mgr.y (mgr.34107) 143 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:50.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:50 smithi089 bash[82331]: cluster 2024-04-13T07:40:49.027731+0000 mgr.y (mgr.34107) 143 : cluster [DBG] pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:51.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[82331]: audit 2024-04-13T07:40:50.548575+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:51.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[82331]: audit 2024-04-13T07:40:50.560121+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:51.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[82331]: audit 2024-04-13T07:40:50.603789+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:40:51.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[82331]: audit 2024-04-13T07:40:51.129285+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:51.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[82331]: audit 2024-04-13T07:40:51.136701+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:51.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[84777]: audit 2024-04-13T07:40:50.548575+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:51.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[84777]: audit 2024-04-13T07:40:50.560121+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:51.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[84777]: audit 2024-04-13T07:40:50.603789+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:40:51.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[84777]: audit 2024-04-13T07:40:51.129285+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:51.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:51 smithi089 bash[84777]: audit 2024-04-13T07:40:51.136701+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:52.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:51 smithi203 bash[68060]: audit 2024-04-13T07:40:50.548575+0000 mon.a (mon.0) 300 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:52.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:51 smithi203 bash[68060]: audit 2024-04-13T07:40:50.560121+0000 mon.a (mon.0) 301 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:52.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:51 smithi203 bash[68060]: audit 2024-04-13T07:40:50.603789+0000 mon.a (mon.0) 302 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:40:52.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:51 smithi203 bash[68060]: audit 2024-04-13T07:40:51.129285+0000 mon.a (mon.0) 303 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:52.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:51 smithi203 bash[68060]: audit 2024-04-13T07:40:51.136701+0000 mon.a (mon.0) 304 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:52.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:52 smithi203 bash[68060]: cluster 2024-04-13T07:40:51.028222+0000 mgr.y (mgr.34107) 144 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:52.765 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:52 smithi203 bash[60252]: ts=2024-04-13T07:40:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:53.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:52 smithi089 bash[82331]: cluster 2024-04-13T07:40:51.028222+0000 mgr.y (mgr.34107) 144 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:53.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:52 smithi089 bash[84777]: cluster 2024-04-13T07:40:51.028222+0000 mgr.y (mgr.34107) 144 : cluster [DBG] pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:53.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:40:53 smithi203 bash[60252]: ts=2024-04-13T07:40:53.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:40:55.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:54 smithi203 bash[68060]: cluster 2024-04-13T07:40:53.029367+0000 mgr.y (mgr.34107) 145 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:55.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:54 smithi089 bash[84777]: cluster 2024-04-13T07:40:53.029367+0000 mgr.y (mgr.34107) 145 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:55.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:54 smithi089 bash[82331]: cluster 2024-04-13T07:40:53.029367+0000 mgr.y (mgr.34107) 145 : cluster [DBG] pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:56.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:40:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:40:55] "GET /metrics HTTP/1.1" 200 33888 "" "Prometheus/2.43.0" 2024-04-13T07:40:56.562 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:40:56 smithi203 bash[49777]: logger=sqlstore.transactions t=2024-04-13T07:40:56.277222251Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-13T07:40:56.562 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:40:56 smithi203 bash[49777]: logger=cleanup t=2024-04-13T07:40:56.367440998Z level=info msg="Completed cleanup jobs" duration=101.643339ms 2024-04-13T07:40:57.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:56 smithi203 bash[68060]: cluster 2024-04-13T07:40:55.030086+0000 mgr.y (mgr.34107) 146 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:56 smithi089 bash[82331]: cluster 2024-04-13T07:40:55.030086+0000 mgr.y (mgr.34107) 146 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:57.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:56 smithi089 bash[84777]: cluster 2024-04-13T07:40:55.030086+0000 mgr.y (mgr.34107) 146 : cluster [DBG] pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:57 smithi203 bash[68060]: audit 2024-04-13T07:40:56.930532+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:57 smithi203 bash[68060]: audit 2024-04-13T07:40:56.942512+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:57 smithi203 bash[68060]: cluster 2024-04-13T07:40:57.030799+0000 mgr.y (mgr.34107) 147 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:57 smithi203 bash[68060]: audit 2024-04-13T07:40:57.063143+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:57 smithi203 bash[68060]: audit 2024-04-13T07:40:57.621947+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:57 smithi203 bash[68060]: audit 2024-04-13T07:40:57.623775+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:40:58.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:40:57 smithi203 bash[68060]: audit 2024-04-13T07:40:57.633964+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:58.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[84777]: audit 2024-04-13T07:40:56.930532+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:58.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[84777]: audit 2024-04-13T07:40:56.942512+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:58.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[84777]: cluster 2024-04-13T07:40:57.030799+0000 mgr.y (mgr.34107) 147 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[84777]: audit 2024-04-13T07:40:57.063143+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[84777]: audit 2024-04-13T07:40:57.621947+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[84777]: audit 2024-04-13T07:40:57.623775+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[84777]: audit 2024-04-13T07:40:57.633964+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[82331]: audit 2024-04-13T07:40:56.930532+0000 mon.a (mon.0) 305 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[82331]: audit 2024-04-13T07:40:56.942512+0000 mon.a (mon.0) 306 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[82331]: cluster 2024-04-13T07:40:57.030799+0000 mgr.y (mgr.34107) 147 : cluster [DBG] pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[82331]: audit 2024-04-13T07:40:57.063143+0000 mon.a (mon.0) 307 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[82331]: audit 2024-04-13T07:40:57.621947+0000 mon.a (mon.0) 308 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:40:58.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[82331]: audit 2024-04-13T07:40:57.623775+0000 mon.a (mon.0) 309 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:40:58.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:40:57 smithi089 bash[82331]: audit 2024-04-13T07:40:57.633964+0000 mon.a (mon.0) 310 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:00.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:00 smithi203 bash[68060]: cluster 2024-04-13T07:40:59.031868+0000 mgr.y (mgr.34107) 148 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:00.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:00 smithi089 bash[82331]: cluster 2024-04-13T07:40:59.031868+0000 mgr.y (mgr.34107) 148 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:00.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:00 smithi089 bash[84777]: cluster 2024-04-13T07:40:59.031868+0000 mgr.y (mgr.34107) 148 : cluster [DBG] pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:02.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:02 smithi203 bash[68060]: cluster 2024-04-13T07:41:01.032502+0000 mgr.y (mgr.34107) 149 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:02.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:41:02 smithi203 bash[60252]: ts=2024-04-13T07:41:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:41:02.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:02 smithi089 bash[82331]: cluster 2024-04-13T07:41:01.032502+0000 mgr.y (mgr.34107) 149 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:02.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:02 smithi089 bash[84777]: cluster 2024-04-13T07:41:01.032502+0000 mgr.y (mgr.34107) 149 : cluster [DBG] pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:03.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:41:03 smithi203 bash[60252]: ts=2024-04-13T07:41:03.508Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:41:04.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:04 smithi203 bash[68060]: cluster 2024-04-13T07:41:03.033506+0000 mgr.y (mgr.34107) 150 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:04.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:04 smithi089 bash[84777]: cluster 2024-04-13T07:41:03.033506+0000 mgr.y (mgr.34107) 150 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:04.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:04 smithi089 bash[82331]: cluster 2024-04-13T07:41:03.033506+0000 mgr.y (mgr.34107) 150 : cluster [DBG] pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:06.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:41:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:41:05] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2024-04-13T07:41:06.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:06 smithi089 bash[84777]: cluster 2024-04-13T07:41:05.034122+0000 mgr.y (mgr.34107) 151 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:06.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:06 smithi089 bash[82331]: cluster 2024-04-13T07:41:05.034122+0000 mgr.y (mgr.34107) 151 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:06.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:06 smithi203 bash[68060]: cluster 2024-04-13T07:41:05.034122+0000 mgr.y (mgr.34107) 151 : cluster [DBG] pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:08.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:08 smithi203 bash[68060]: cluster 2024-04-13T07:41:07.034744+0000 mgr.y (mgr.34107) 152 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:08 smithi089 bash[82331]: cluster 2024-04-13T07:41:07.034744+0000 mgr.y (mgr.34107) 152 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:08.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:08 smithi089 bash[84777]: cluster 2024-04-13T07:41:07.034744+0000 mgr.y (mgr.34107) 152 : cluster [DBG] pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:10.457 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:10 smithi089 bash[82331]: cluster 2024-04-13T07:41:09.035822+0000 mgr.y (mgr.34107) 153 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:10.458 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:10 smithi089 bash[84777]: cluster 2024-04-13T07:41:09.035822+0000 mgr.y (mgr.34107) 153 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:10.459 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch upgrade resume' 2024-04-13T07:41:10.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:10 smithi203 bash[68060]: cluster 2024-04-13T07:41:09.035822+0000 mgr.y (mgr.34107) 153 : cluster [DBG] pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:12.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:12 smithi203 bash[68060]: cluster 2024-04-13T07:41:11.036436+0000 mgr.y (mgr.34107) 154 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:12 smithi203 bash[68060]: audit 2024-04-13T07:41:12.063886+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:12.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:41:12 smithi203 bash[60252]: ts=2024-04-13T07:41:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:41:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:12 smithi089 bash[82331]: cluster 2024-04-13T07:41:11.036436+0000 mgr.y (mgr.34107) 154 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:12 smithi089 bash[82331]: audit 2024-04-13T07:41:12.063886+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:12.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:12 smithi089 bash[84777]: cluster 2024-04-13T07:41:11.036436+0000 mgr.y (mgr.34107) 154 : cluster [DBG] pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:12.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:12 smithi089 bash[84777]: audit 2024-04-13T07:41:12.063886+0000 mon.a (mon.0) 311 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:13.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:41:13 smithi203 bash[60252]: ts=2024-04-13T07:41:13.509Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=nodes name=CephNodeDiskspaceWarning index=4 msg="Evaluating rule failed" rule="alert: CephNodeDiskspaceWarning\nexpr: predict_linear(node_filesystem_free_bytes{device=~\"/.*\"}[2d], 3600 * 24 * 5)\n * on (instance) group_left (nodename) node_uname_info < 0\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.8.4\n severity: warning\n type: ceph_default\nannotations:\n description: Mountpoint {{ $labels.mountpoint }} on {{ $labels.nodename }} will\n be full in less than 5 days based on the 48 hour trailing fill rate.\n summary: Host filesystem free space is getting low\n" err="found duplicate series for the match group {instance=\"smithi089\"} on the right hand-side of the operation: [{__name__=\"node_uname_info\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}, {__name__=\"node_uname_info\", cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", domainname=\"(none)\", instance=\"smithi089\", job=\"node\", machine=\"x86_64\", nodename=\"smithi089\", release=\"5.15.0-102-generic\", sysname=\"Linux\", version=\"#112-Ubuntu SMP Tue Mar 5 16:50:32 UTC 2024\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:41:14.518 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:14 smithi203 bash[68060]: cluster 2024-04-13T07:41:13.037481+0000 mgr.y (mgr.34107) 155 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:14.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:14 smithi089 bash[82331]: cluster 2024-04-13T07:41:13.037481+0000 mgr.y (mgr.34107) 155 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:14.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:14 smithi089 bash[84777]: cluster 2024-04-13T07:41:13.037481+0000 mgr.y (mgr.34107) 155 : cluster [DBG] pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:15.203 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:41:16.116 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:41:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:41:15] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2024-04-13T07:41:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:16 smithi203 bash[68060]: cluster 2024-04-13T07:41:15.038098+0000 mgr.y (mgr.34107) 156 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:16 smithi089 bash[84777]: cluster 2024-04-13T07:41:15.038098+0000 mgr.y (mgr.34107) 156 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:16 smithi089 bash[82331]: cluster 2024-04-13T07:41:15.038098+0000 mgr.y (mgr.34107) 156 : cluster [DBG] pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:17.872 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:41:18.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:18 smithi089 bash[82331]: cluster 2024-04-13T07:41:17.038741+0000 mgr.y (mgr.34107) 157 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:18.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:18 smithi089 bash[82331]: audit 2024-04-13T07:41:17.878821+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:18.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:18 smithi089 bash[82331]: audit 2024-04-13T07:41:17.881284+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:18.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:18 smithi089 bash[84777]: cluster 2024-04-13T07:41:17.038741+0000 mgr.y (mgr.34107) 157 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:18.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:18 smithi089 bash[84777]: audit 2024-04-13T07:41:17.878821+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:18.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:18 smithi089 bash[84777]: audit 2024-04-13T07:41:17.881284+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:18.493 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'echo wait for all mons to upgrade' 2024-04-13T07:41:18.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:18 smithi203 bash[68060]: cluster 2024-04-13T07:41:17.038741+0000 mgr.y (mgr.34107) 157 : cluster [DBG] pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:18.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:18 smithi203 bash[68060]: audit 2024-04-13T07:41:17.878821+0000 mon.a (mon.0) 312 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:18.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:18 smithi203 bash[68060]: audit 2024-04-13T07:41:17.881284+0000 mon.a (mon.0) 313 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:19.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:19 smithi203 bash[68060]: audit 2024-04-13T07:41:17.873572+0000 mgr.y (mgr.34107) 158 : audit [DBG] from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:19.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:19 smithi203 bash[68060]: cephadm 2024-04-13T07:41:17.873954+0000 mgr.y (mgr.34107) 159 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:41:19.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:19 smithi089 bash[82331]: audit 2024-04-13T07:41:17.873572+0000 mgr.y (mgr.34107) 158 : audit [DBG] from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:19.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:19 smithi089 bash[82331]: cephadm 2024-04-13T07:41:17.873954+0000 mgr.y (mgr.34107) 159 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:41:19.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:19 smithi089 bash[84777]: audit 2024-04-13T07:41:17.873572+0000 mgr.y (mgr.34107) 158 : audit [DBG] from='client.34191 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:19.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:19 smithi089 bash[84777]: cephadm 2024-04-13T07:41:17.873954+0000 mgr.y (mgr.34107) 159 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:41:20.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:20 smithi203 bash[68060]: cluster 2024-04-13T07:41:19.039832+0000 mgr.y (mgr.34107) 160 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:20.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:20 smithi089 bash[82331]: cluster 2024-04-13T07:41:19.039832+0000 mgr.y (mgr.34107) 160 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:20.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:20 smithi089 bash[84777]: cluster 2024-04-13T07:41:19.039832+0000 mgr.y (mgr.34107) 160 : cluster [DBG] pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:22.596 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:41:22 smithi203 bash[60252]: ts=2024-04-13T07:41:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:41:23.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:22 smithi203 bash[68060]: cluster 2024-04-13T07:41:21.040402+0000 mgr.y (mgr.34107) 161 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:23.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:22 smithi089 bash[82331]: cluster 2024-04-13T07:41:21.040402+0000 mgr.y (mgr.34107) 161 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:23.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:22 smithi089 bash[84777]: cluster 2024-04-13T07:41:21.040402+0000 mgr.y (mgr.34107) 161 : cluster [DBG] pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:23.227 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:41:25.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:24 smithi203 bash[68060]: cluster 2024-04-13T07:41:23.041419+0000 mgr.y (mgr.34107) 162 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:24 smithi203 bash[68060]: audit 2024-04-13T07:41:23.930969+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:24 smithi203 bash[68060]: audit 2024-04-13T07:41:23.942624+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:24 smithi089 bash[84777]: cluster 2024-04-13T07:41:23.041419+0000 mgr.y (mgr.34107) 162 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:25.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:24 smithi089 bash[84777]: audit 2024-04-13T07:41:23.930969+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:24 smithi089 bash[84777]: audit 2024-04-13T07:41:23.942624+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:24 smithi089 bash[82331]: cluster 2024-04-13T07:41:23.041419+0000 mgr.y (mgr.34107) 162 : cluster [DBG] pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:24 smithi089 bash[82331]: audit 2024-04-13T07:41:23.930969+0000 mon.a (mon.0) 314 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:24 smithi089 bash[82331]: audit 2024-04-13T07:41:23.942624+0000 mon.a (mon.0) 315 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.932 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:41:25] "GET /metrics HTTP/1.1" 200 33885 "" "Prometheus/2.43.0" 2024-04-13T07:41:25.932 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.617311+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.619051+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.628891+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.728099+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.732452+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.735933+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: cephadm 2024-04-13T07:41:24.737003+0000 mgr.y (mgr.34107) 163 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.741816+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.744273+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.749194+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.752040+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.756719+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.758940+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.762975+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-13T07:41:25.933 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.766375+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.771790+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[84777]: audit 2024-04-13T07:41:24.774754+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.617311+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.619051+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.628891+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.728099+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.732452+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.735933+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: cephadm 2024-04-13T07:41:24.737003+0000 mgr.y (mgr.34107) 163 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.741816+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.744273+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-13T07:41:25.934 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.749194+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-13T07:41:25.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.752040+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-13T07:41:25.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.756719+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-13T07:41:25.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.758940+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-13T07:41:25.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.762975+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-13T07:41:25.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.766375+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:25.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.771790+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:25.935 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:25 smithi089 bash[82331]: audit 2024-04-13T07:41:24.774754+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-13T07:41:26.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.617311+0000 mon.a (mon.0) 316 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:26.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.619051+0000 mon.a (mon.0) 317 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:26.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.628891+0000 mon.a (mon.0) 318 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:26.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.728099+0000 mon.a (mon.0) 319 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:26.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.732452+0000 mon.a (mon.0) 320 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:26.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.735933+0000 mon.a (mon.0) 321 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:26.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: cephadm 2024-04-13T07:41:24.737003+0000 mgr.y (mgr.34107) 163 : cephadm [INF] Upgrade: Setting container_image for all mon 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.741816+0000 mon.a (mon.0) 322 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.744273+0000 mon.a (mon.0) 323 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.749194+0000 mon.a (mon.0) 324 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.752040+0000 mon.a (mon.0) 325 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.756719+0000 mon.a (mon.0) 326 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.758940+0000 mon.a (mon.0) 327 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.762975+0000 mon.a (mon.0) 328 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.766375+0000 mon.a (mon.0) 329 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.771790+0000 mon.a (mon.0) 330 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:26.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:25 smithi203 bash[68060]: audit 2024-04-13T07:41:24.774754+0000 mon.a (mon.0) 331 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-13T07:41:26.172 INFO:teuthology.orchestra.run.smithi089.stdout:wait for all mons to upgrade 2024-04-13T07:41:26.893 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:26 smithi203 bash[68060]: cephadm 2024-04-13T07:41:24.767501+0000 mgr.y (mgr.34107) 164 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-04-13T07:41:26.893 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:26 smithi203 bash[68060]: audit 2024-04-13T07:41:24.775054+0000 mgr.y (mgr.34107) 165 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-13T07:41:26.893 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:26 smithi203 bash[68060]: cephadm 2024-04-13T07:41:24.776133+0000 mgr.y (mgr.34107) 166 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-04-13T07:41:26.893 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:26 smithi203 bash[68060]: cluster 2024-04-13T07:41:25.042111+0000 mgr.y (mgr.34107) 167 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:27.076 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:26 smithi089 bash[84777]: cephadm 2024-04-13T07:41:24.767501+0000 mgr.y (mgr.34107) 164 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-04-13T07:41:27.076 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:26 smithi089 bash[84777]: audit 2024-04-13T07:41:24.775054+0000 mgr.y (mgr.34107) 165 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-13T07:41:27.076 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:26 smithi089 bash[84777]: cephadm 2024-04-13T07:41:24.776133+0000 mgr.y (mgr.34107) 166 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-04-13T07:41:27.076 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:26 smithi089 bash[84777]: cluster 2024-04-13T07:41:25.042111+0000 mgr.y (mgr.34107) 167 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:27.076 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:26 smithi089 bash[82331]: cephadm 2024-04-13T07:41:24.767501+0000 mgr.y (mgr.34107) 164 : cephadm [INF] Upgrade: Setting container_image for all crash 2024-04-13T07:41:27.076 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:26 smithi089 bash[82331]: audit 2024-04-13T07:41:24.775054+0000 mgr.y (mgr.34107) 165 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2024-04-13T07:41:27.076 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:26 smithi089 bash[82331]: cephadm 2024-04-13T07:41:24.776133+0000 mgr.y (mgr.34107) 166 : cephadm [INF] Upgrade: osd.0 is safe to restart 2024-04-13T07:41:27.077 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:26 smithi089 bash[82331]: cluster 2024-04-13T07:41:25.042111+0000 mgr.y (mgr.34107) 167 : cluster [DBG] pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:27.468 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2024-04-13T07:41:27.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:27 smithi089 bash[84777]: audit 2024-04-13T07:41:27.064056+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:27.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:27 smithi089 bash[84777]: audit 2024-04-13T07:41:27.509888+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:27.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:27 smithi089 bash[84777]: audit 2024-04-13T07:41:27.512756+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:41:27.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:27 smithi089 bash[84777]: audit 2024-04-13T07:41:27.513608+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:27 smithi089 bash[82331]: audit 2024-04-13T07:41:27.064056+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:27 smithi089 bash[82331]: audit 2024-04-13T07:41:27.509888+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:27 smithi089 bash[82331]: audit 2024-04-13T07:41:27.512756+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:41:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:27 smithi089 bash[82331]: audit 2024-04-13T07:41:27.513608+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:28.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:27 smithi203 bash[68060]: audit 2024-04-13T07:41:27.064056+0000 mon.a (mon.0) 332 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:28.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:27 smithi203 bash[68060]: audit 2024-04-13T07:41:27.509888+0000 mon.a (mon.0) 333 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:28.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:27 smithi203 bash[68060]: audit 2024-04-13T07:41:27.512756+0000 mon.a (mon.0) 334 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2024-04-13T07:41:28.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:27 smithi203 bash[68060]: audit 2024-04-13T07:41:27.513608+0000 mon.a (mon.0) 335 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:28.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:28 smithi089 bash[82331]: cluster 2024-04-13T07:41:27.042726+0000 mgr.y (mgr.34107) 168 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:28.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:28 smithi089 bash[82331]: cephadm 2024-04-13T07:41:27.505126+0000 mgr.y (mgr.34107) 169 : cephadm [INF] Upgrade: Updating osd.0 2024-04-13T07:41:28.905 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:28 smithi089 bash[82331]: cephadm 2024-04-13T07:41:27.516691+0000 mgr.y (mgr.34107) 170 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-04-13T07:41:28.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:28 smithi089 bash[84777]: cluster 2024-04-13T07:41:27.042726+0000 mgr.y (mgr.34107) 168 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:28.905 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:28 smithi089 bash[84777]: cephadm 2024-04-13T07:41:27.505126+0000 mgr.y (mgr.34107) 169 : cephadm [INF] Upgrade: Updating osd.0 2024-04-13T07:41:28.906 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:28 smithi089 bash[84777]: cephadm 2024-04-13T07:41:27.516691+0000 mgr.y (mgr.34107) 170 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-04-13T07:41:29.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:28 smithi203 bash[68060]: cluster 2024-04-13T07:41:27.042726+0000 mgr.y (mgr.34107) 168 : cluster [DBG] pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:29.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:28 smithi203 bash[68060]: cephadm 2024-04-13T07:41:27.505126+0000 mgr.y (mgr.34107) 169 : cephadm [INF] Upgrade: Updating osd.0 2024-04-13T07:41:29.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:28 smithi203 bash[68060]: cephadm 2024-04-13T07:41:27.516691+0000 mgr.y (mgr.34107) 170 : cephadm [INF] Deploying daemon osd.0 on smithi089 2024-04-13T07:41:31.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:30 smithi203 bash[68060]: cluster 2024-04-13T07:41:29.043872+0000 mgr.y (mgr.34107) 171 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:31.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:30 smithi089 bash[82331]: cluster 2024-04-13T07:41:29.043872+0000 mgr.y (mgr.34107) 171 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:31.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:30 smithi089 bash[84777]: cluster 2024-04-13T07:41:29.043872+0000 mgr.y (mgr.34107) 171 : cluster [DBG] pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:31.635 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:31.635 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:31.636 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:31.636 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:31.636 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:31.637 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:31.637 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:31.637 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:31.637 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:32.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:31 smithi089 systemd[1]: Stopping Ceph osd.0 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:41:32.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:31 smithi089 bash[34548]: debug 2024-04-13T07:41:31.764+0000 7f634fd34700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:41:32.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:31 smithi089 bash[34548]: debug 2024-04-13T07:41:31.764+0000 7f634fd34700 -1 osd.0 92 *** Got signal Terminated *** 2024-04-13T07:41:32.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:31 smithi089 bash[34548]: debug 2024-04-13T07:41:31.764+0000 7f634fd34700 -1 osd.0 92 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:41:32.646 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:41:32 smithi203 bash[60252]: ts=2024-04-13T07:41:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:41:32.719 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:41:33.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:32 smithi203 bash[68060]: cluster 2024-04-13T07:41:31.044527+0000 mgr.y (mgr.34107) 172 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:33.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:32 smithi203 bash[68060]: cluster 2024-04-13T07:41:31.769486+0000 mon.a (mon.0) 336 : cluster [INF] osd.0 marked itself down and dead 2024-04-13T07:41:33.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:32 smithi089 bash[82331]: debug 2024-04-13T07:41:32.624+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:41:33.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:32 smithi089 bash[82331]: cluster 2024-04-13T07:41:31.044527+0000 mgr.y (mgr.34107) 172 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:33.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:32 smithi089 bash[82331]: cluster 2024-04-13T07:41:31.769486+0000 mon.a (mon.0) 336 : cluster [INF] osd.0 marked itself down and dead 2024-04-13T07:41:33.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:32 smithi089 bash[84777]: cluster 2024-04-13T07:41:31.044527+0000 mgr.y (mgr.34107) 172 : cluster [DBG] pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:33.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:32 smithi089 bash[84777]: cluster 2024-04-13T07:41:31.769486+0000 mon.a (mon.0) 336 : cluster [INF] osd.0 marked itself down and dead 2024-04-13T07:41:34.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:33 smithi203 bash[68060]: cluster 2024-04-13T07:41:32.627306+0000 mon.a (mon.0) 337 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:41:34.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:33 smithi203 bash[68060]: cluster 2024-04-13T07:41:32.627376+0000 mon.a (mon.0) 338 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:41:34.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:33 smithi203 bash[68060]: cluster 2024-04-13T07:41:32.638704+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e93: 8 total, 7 up, 8 in 2024-04-13T07:41:34.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:33 smithi203 bash[68060]: cluster 2024-04-13T07:41:33.635779+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e94: 8 total, 7 up, 8 in 2024-04-13T07:41:34.046 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[90254]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-0 2024-04-13T07:41:34.046 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[82331]: cluster 2024-04-13T07:41:32.627306+0000 mon.a (mon.0) 337 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:41:34.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[82331]: cluster 2024-04-13T07:41:32.627376+0000 mon.a (mon.0) 338 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:41:34.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[82331]: cluster 2024-04-13T07:41:32.638704+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e93: 8 total, 7 up, 8 in 2024-04-13T07:41:34.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[82331]: cluster 2024-04-13T07:41:33.635779+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e94: 8 total, 7 up, 8 in 2024-04-13T07:41:34.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[84777]: cluster 2024-04-13T07:41:32.627306+0000 mon.a (mon.0) 337 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:41:34.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[84777]: cluster 2024-04-13T07:41:32.627376+0000 mon.a (mon.0) 338 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:41:34.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[84777]: cluster 2024-04-13T07:41:32.638704+0000 mon.a (mon.0) 339 : cluster [DBG] osdmap e93: 8 total, 7 up, 8 in 2024-04-13T07:41:34.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:33 smithi089 bash[84777]: cluster 2024-04-13T07:41:33.635779+0000 mon.a (mon.0) 340 : cluster [DBG] osdmap e94: 8 total, 7 up, 8 in 2024-04-13T07:41:34.548 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:34 smithi089 bash[90328]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-0 2024-04-13T07:41:34.936 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:34 smithi089 bash[82331]: cluster 2024-04-13T07:41:33.045855+0000 mgr.y (mgr.34107) 173 : cluster [DBG] pgmap v112: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:34.937 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:34 smithi089 bash[82331]: cluster 2024-04-13T07:41:33.642375+0000 mon.a (mon.0) 341 : cluster [WRN] Health check failed: Reduced data availability: 9 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2024-04-13T07:41:34.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:34 smithi089 bash[84777]: cluster 2024-04-13T07:41:33.045855+0000 mgr.y (mgr.34107) 173 : cluster [DBG] pgmap v112: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:34.937 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:34 smithi089 bash[84777]: cluster 2024-04-13T07:41:33.642375+0000 mon.a (mon.0) 341 : cluster [WRN] Health check failed: Reduced data availability: 9 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2024-04-13T07:41:35.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:34 smithi203 bash[68060]: cluster 2024-04-13T07:41:33.045855+0000 mgr.y (mgr.34107) 173 : cluster [DBG] pgmap v112: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:35.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:34 smithi203 bash[68060]: cluster 2024-04-13T07:41:33.642375+0000 mon.a (mon.0) 341 : cluster [WRN] Health check failed: Reduced data availability: 9 pgs inactive, 17 pgs peering (PG_AVAILABILITY) 2024-04-13T07:41:36.011 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:41:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:41:35] "GET /metrics HTTP/1.1" 200 33822 "" "Prometheus/2.43.0" 2024-04-13T07:41:36.916 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:36 smithi089 bash[82331]: cluster 2024-04-13T07:41:35.046827+0000 mgr.y (mgr.34107) 174 : cluster [DBG] pgmap v114: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:36.916 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:36 smithi089 bash[84777]: cluster 2024-04-13T07:41:35.046827+0000 mgr.y (mgr.34107) 174 : cluster [DBG] pgmap v114: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:37.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:36 smithi203 bash[68060]: cluster 2024-04-13T07:41:35.046827+0000 mgr.y (mgr.34107) 174 : cluster [DBG] pgmap v114: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:37.192 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.193 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.193 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.193 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.194 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.194 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:36 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.0.service: Deactivated successfully. 2024-04-13T07:41:37.194 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:36 smithi089 systemd[1]: Stopped Ceph osd.0 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:41:37.194 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.195 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.195 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.195 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:41:37.548 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:37 smithi089 systemd[1]: Started Ceph osd.0 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:41:37.655 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev)": 3 2024-04-13T07:41:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:38 smithi203 bash[68060]: cluster 2024-04-13T07:41:37.047385+0000 mgr.y (mgr.34107) 175 : cluster [DBG] pgmap v115: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:38 smithi203 bash[68060]: audit 2024-04-13T07:41:37.246828+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:38 smithi203 bash[68060]: audit 2024-04-13T07:41:37.255498+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:38 smithi203 bash[68060]: audit 2024-04-13T07:41:37.301954+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:38 smithi203 bash[68060]: audit 2024-04-13T07:41:37.648080+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.89:0/1328672453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:41:38.517 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[82331]: cluster 2024-04-13T07:41:37.047385+0000 mgr.y (mgr.34107) 175 : cluster [DBG] pgmap v115: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[82331]: audit 2024-04-13T07:41:37.246828+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[82331]: audit 2024-04-13T07:41:37.255498+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[82331]: audit 2024-04-13T07:41:37.301954+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[82331]: audit 2024-04-13T07:41:37.648080+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.89:0/1328672453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[84777]: cluster 2024-04-13T07:41:37.047385+0000 mgr.y (mgr.34107) 175 : cluster [DBG] pgmap v115: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 252 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[84777]: audit 2024-04-13T07:41:37.246828+0000 mon.a (mon.0) 342 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[84777]: audit 2024-04-13T07:41:37.255498+0000 mon.a (mon.0) 343 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[84777]: audit 2024-04-13T07:41:37.301954+0000 mon.a (mon.0) 344 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:38.518 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:38 smithi089 bash[84777]: audit 2024-04-13T07:41:37.648080+0000 mon.c (mon.2) 4 : audit [DBG] from='client.? 172.21.15.89:0/1328672453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2024-04-13T07:41:38.519 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch ps' 2024-04-13T07:41:39.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[82331]: cluster 2024-04-13T07:41:39.245409+0000 mon.a (mon.0) 345 : cluster [WRN] Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-13T07:41:39.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[82331]: cluster 2024-04-13T07:41:39.245445+0000 mon.a (mon.0) 346 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 17 pgs peering) 2024-04-13T07:41:39.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[84777]: cluster 2024-04-13T07:41:39.245409+0000 mon.a (mon.0) 345 : cluster [WRN] Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-13T07:41:39.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[84777]: cluster 2024-04-13T07:41:39.245445+0000 mon.a (mon.0) 346 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 17 pgs peering) 2024-04-13T07:41:39.549 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[90601]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-13T07:41:39.549 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[90601]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-0 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-13T07:41:39.549 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[90601]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-13T07:41:39.550 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[90601]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-13T07:41:39.550 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[90601]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-0/block 2024-04-13T07:41:39.550 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[90601]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2024-04-13T07:41:39.550 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:39 smithi089 bash[90601]: --> ceph-volume raw activate successful for osd ID: 0 2024-04-13T07:41:39.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:39 smithi203 bash[68060]: cluster 2024-04-13T07:41:39.245409+0000 mon.a (mon.0) 345 : cluster [WRN] Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-13T07:41:39.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:39 smithi203 bash[68060]: cluster 2024-04-13T07:41:39.245445+0000 mon.a (mon.0) 346 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 17 pgs peering) 2024-04-13T07:41:40.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:40 smithi089 bash[82331]: cluster 2024-04-13T07:41:39.048060+0000 mgr.y (mgr.34107) 176 : cluster [DBG] pgmap v116: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:40.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:40 smithi089 bash[84777]: cluster 2024-04-13T07:41:39.048060+0000 mgr.y (mgr.34107) 176 : cluster [DBG] pgmap v116: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:40.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:40 smithi203 bash[68060]: cluster 2024-04-13T07:41:39.048060+0000 mgr.y (mgr.34107) 176 : cluster [DBG] pgmap v116: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:41.048 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:40 smithi089 bash[90890]: debug 2024-04-13T07:41:40.720+0000 7fb7cf41e700 1 -- 172.21.15.89:0/3791544969 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55767f627680 con 0x55767f664c00 2024-04-13T07:41:41.548 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:41 smithi089 bash[90890]: debug 2024-04-13T07:41:41.252+0000 7fb7d85b67c0 -1 Falling back to public interface 2024-04-13T07:41:42.420 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:41:42.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:42 smithi089 bash[84777]: cluster 2024-04-13T07:41:41.048733+0000 mgr.y (mgr.34107) 177 : cluster [DBG] pgmap v117: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:42.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:42 smithi089 bash[84777]: audit 2024-04-13T07:41:42.070681+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:42.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:42 smithi089 bash[84777]: audit 2024-04-13T07:41:42.072931+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:42.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:42 smithi089 bash[82331]: cluster 2024-04-13T07:41:41.048733+0000 mgr.y (mgr.34107) 177 : cluster [DBG] pgmap v117: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:42.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:42 smithi089 bash[82331]: audit 2024-04-13T07:41:42.070681+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:42.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:42 smithi089 bash[82331]: audit 2024-04-13T07:41:42.072931+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:42.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:42 smithi203 bash[68060]: cluster 2024-04-13T07:41:41.048733+0000 mgr.y (mgr.34107) 177 : cluster [DBG] pgmap v117: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:42.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:42 smithi203 bash[68060]: audit 2024-04-13T07:41:42.070681+0000 mon.a (mon.0) 347 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:42.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:42 smithi203 bash[68060]: audit 2024-04-13T07:41:42.072931+0000 mon.a (mon.0) 348 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:42.765 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:41:42 smithi203 bash[60252]: ts=2024-04-13T07:41:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:41:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[82331]: cluster 2024-04-13T07:41:43.049350+0000 mgr.y (mgr.34107) 178 : cluster [DBG] pgmap v118: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[82331]: audit 2024-04-13T07:41:43.192503+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[82331]: audit 2024-04-13T07:41:43.198007+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[82331]: audit 2024-04-13T07:41:44.188986+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[84777]: cluster 2024-04-13T07:41:43.049350+0000 mgr.y (mgr.34107) 178 : cluster [DBG] pgmap v118: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:44.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[84777]: audit 2024-04-13T07:41:43.192503+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[84777]: audit 2024-04-13T07:41:43.198007+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[84777]: audit 2024-04-13T07:41:44.188986+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:44 smithi203 bash[68060]: cluster 2024-04-13T07:41:43.049350+0000 mgr.y (mgr.34107) 178 : cluster [DBG] pgmap v118: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:44 smithi203 bash[68060]: audit 2024-04-13T07:41:43.192503+0000 mon.a (mon.0) 349 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:44 smithi203 bash[68060]: audit 2024-04-13T07:41:43.198007+0000 mon.a (mon.0) 350 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:44 smithi203 bash[68060]: audit 2024-04-13T07:41:44.188986+0000 mon.a (mon.0) 351 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:44.676 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:44 smithi089 bash[90890]: debug 2024-04-13T07:41:44.344+0000 7fb7d85b67c0 -1 osd.0 92 log_to_monitors true 2024-04-13T07:41:45.047 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-13T07:41:45.047 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (11m) 1s ago 11m 13.9M - 0.25.0 c8568f914cd2 d39c47986efe 2024-04-13T07:41:45.047 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi203 *:3000 running (11m) 21s ago 11m 79.0M - 9.4.7 954c08fa6188 4496ec520de5 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.pccnvq smithi089 running (9m) 1s ago 9m 14.9M - 18.2.2-1217-g7d36322c 6ecd1226aa2f 07167bfc6e5f 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi203.sjspdy smithi203 running (9m) 21s ago 9m 16.9M - 18.2.2-1217-g7d36322c 6ecd1226aa2f e100a52f8447 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi203 *:8443,9283,8765 running (4m) 21s ago 19m 437M - 19.0.0-2905-g5f22a497 c4b45bf0c946 54a5a6c61bc7 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (5m) 1s ago 22m 494M - 19.0.0-2905-g5f22a497 c4b45bf0c946 f20e719fbf68 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (4m) 1s ago 22m 45.4M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 d2e86a9f9ec2 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi203 running (111s) 21s ago 20m 34.6M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 fb3dac884f39 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (2m) 1s ago 20m 34.5M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 0345f038f9be 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (11m) 1s ago 11m 8779k - 1.5.0 0da6a335fe13 04462903d931 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi203 *:9100 running (11m) 21s ago 11m 8823k - 1.5.0 0da6a335fe13 a79084a2b37c 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (5s) 1s ago 19m - 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 dffd1ec4051b 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (18m) 1s ago 18m 53.1M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 58fb3beb85cc 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (17m) 1s ago 17m 50.8M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f e020a590fd83 2024-04-13T07:41:45.048 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (16m) 1s ago 16m 47.2M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 1c6772886276 2024-04-13T07:41:45.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi203 running (15m) 21s ago 15m 55.4M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 28af48f1a163 2024-04-13T07:41:45.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi203 running (14m) 21s ago 15m 48.8M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f afe79fa4d7d0 2024-04-13T07:41:45.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi203 running (14m) 21s ago 14m 47.8M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 8913a43d05b8 2024-04-13T07:41:45.049 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi203 running (13m) 21s ago 13m 63.6M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 01ed33979f80 2024-04-13T07:41:45.049 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi203 *:9095 running (4m) 21s ago 12m 48.7M - 2.43.0 a07b618ecd1d 50a6268920e0 2024-04-13T07:41:45.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:45 smithi203 bash[68060]: audit 2024-04-13T07:41:44.194792+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:45.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:45 smithi203 bash[68060]: audit 2024-04-13T07:41:44.352275+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:41:45.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:45 smithi203 bash[68060]: audit 2024-04-13T07:41:44.352577+0000 mon.a (mon.0) 353 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:41:45.548 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:41:45 smithi089 bash[90890]: debug 2024-04-13T07:41:45.263+0000 7fb7c9a0e700 -1 osd.0 92 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-13T07:41:45.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:45 smithi089 bash[82331]: audit 2024-04-13T07:41:44.194792+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:45.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:45 smithi089 bash[82331]: audit 2024-04-13T07:41:44.352275+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:41:45.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:45 smithi089 bash[82331]: audit 2024-04-13T07:41:44.352577+0000 mon.a (mon.0) 353 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:41:45.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:45 smithi089 bash[84777]: audit 2024-04-13T07:41:44.194792+0000 mon.a (mon.0) 352 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:45.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:45 smithi089 bash[84777]: audit 2024-04-13T07:41:44.352275+0000 mon.c (mon.2) 5 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:41:45.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:45 smithi089 bash[84777]: audit 2024-04-13T07:41:44.352577+0000 mon.a (mon.0) 353 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2024-04-13T07:41:45.729 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch upgrade pause' 2024-04-13T07:41:46.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:41:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:41:45] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2024-04-13T07:41:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:46 smithi203 bash[68060]: audit 2024-04-13T07:41:45.040174+0000 mgr.y (mgr.34107) 179 : audit [DBG] from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:46 smithi203 bash[68060]: cluster 2024-04-13T07:41:45.049991+0000 mgr.y (mgr.34107) 180 : cluster [DBG] pgmap v119: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:46 smithi203 bash[68060]: audit 2024-04-13T07:41:45.209608+0000 mon.a (mon.0) 354 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-13T07:41:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:46 smithi203 bash[68060]: cluster 2024-04-13T07:41:45.219984+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e95: 8 total, 7 up, 8 in 2024-04-13T07:41:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:46 smithi203 bash[68060]: audit 2024-04-13T07:41:45.220508+0000 mon.c (mon.2) 6 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:41:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:46 smithi203 bash[68060]: audit 2024-04-13T07:41:45.220947+0000 mon.a (mon.0) 356 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:41:46.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[82331]: debug 2024-04-13T07:41:46.203+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:41:46.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[82331]: audit 2024-04-13T07:41:45.040174+0000 mgr.y (mgr.34107) 179 : audit [DBG] from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:46.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[82331]: cluster 2024-04-13T07:41:45.049991+0000 mgr.y (mgr.34107) 180 : cluster [DBG] pgmap v119: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[82331]: audit 2024-04-13T07:41:45.209608+0000 mon.a (mon.0) 354 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[82331]: cluster 2024-04-13T07:41:45.219984+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e95: 8 total, 7 up, 8 in 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[82331]: audit 2024-04-13T07:41:45.220508+0000 mon.c (mon.2) 6 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[82331]: audit 2024-04-13T07:41:45.220947+0000 mon.a (mon.0) 356 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[84777]: audit 2024-04-13T07:41:45.040174+0000 mgr.y (mgr.34107) 179 : audit [DBG] from='client.44138 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[84777]: cluster 2024-04-13T07:41:45.049991+0000 mgr.y (mgr.34107) 180 : cluster [DBG] pgmap v119: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[84777]: audit 2024-04-13T07:41:45.209608+0000 mon.a (mon.0) 354 : audit [INF] from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[84777]: cluster 2024-04-13T07:41:45.219984+0000 mon.a (mon.0) 355 : cluster [DBG] osdmap e95: 8 total, 7 up, 8 in 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[84777]: audit 2024-04-13T07:41:45.220508+0000 mon.c (mon.2) 6 : audit [INF] from='osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:41:46.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:46 smithi089 bash[84777]: audit 2024-04-13T07:41:45.220947+0000 mon.a (mon.0) 356 : audit [INF] from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:41:47.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:47 smithi203 bash[68060]: cluster 2024-04-13T07:41:46.210541+0000 mon.a (mon.0) 357 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:41:47.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:47 smithi203 bash[68060]: cluster 2024-04-13T07:41:46.210612+0000 mon.a (mon.0) 358 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:41:47.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:47 smithi203 bash[68060]: cluster 2024-04-13T07:41:46.221598+0000 mon.a (mon.0) 359 : cluster [INF] osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744] boot 2024-04-13T07:41:47.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:47 smithi203 bash[68060]: cluster 2024-04-13T07:41:46.221714+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-04-13T07:41:47.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:47 smithi203 bash[68060]: audit 2024-04-13T07:41:46.222628+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:41:47.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:47 smithi203 bash[68060]: cluster 2024-04-13T07:41:47.221835+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-04-13T07:41:47.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[82331]: cluster 2024-04-13T07:41:46.210541+0000 mon.a (mon.0) 357 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:41:47.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[82331]: cluster 2024-04-13T07:41:46.210612+0000 mon.a (mon.0) 358 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:41:47.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[82331]: cluster 2024-04-13T07:41:46.221598+0000 mon.a (mon.0) 359 : cluster [INF] osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744] boot 2024-04-13T07:41:47.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[82331]: cluster 2024-04-13T07:41:46.221714+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-04-13T07:41:47.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[82331]: audit 2024-04-13T07:41:46.222628+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:41:47.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[82331]: cluster 2024-04-13T07:41:47.221835+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-04-13T07:41:47.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[84777]: cluster 2024-04-13T07:41:46.210541+0000 mon.a (mon.0) 357 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:41:47.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[84777]: cluster 2024-04-13T07:41:46.210612+0000 mon.a (mon.0) 358 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:41:47.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[84777]: cluster 2024-04-13T07:41:46.221598+0000 mon.a (mon.0) 359 : cluster [INF] osd.0 [v2:172.21.15.89:6802/3498175744,v1:172.21.15.89:6803/3498175744] boot 2024-04-13T07:41:47.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[84777]: cluster 2024-04-13T07:41:46.221714+0000 mon.a (mon.0) 360 : cluster [DBG] osdmap e96: 8 total, 8 up, 8 in 2024-04-13T07:41:47.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[84777]: audit 2024-04-13T07:41:46.222628+0000 mon.a (mon.0) 361 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2024-04-13T07:41:47.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:47 smithi089 bash[84777]: cluster 2024-04-13T07:41:47.221835+0000 mon.a (mon.0) 362 : cluster [DBG] osdmap e97: 8 total, 8 up, 8 in 2024-04-13T07:41:48.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:48 smithi203 bash[68060]: cluster 2024-04-13T07:41:47.050751+0000 mgr.y (mgr.34107) 181 : cluster [DBG] pgmap v122: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:48.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:48 smithi089 bash[82331]: cluster 2024-04-13T07:41:47.050751+0000 mgr.y (mgr.34107) 181 : cluster [DBG] pgmap v122: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:48.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:48 smithi089 bash[84777]: cluster 2024-04-13T07:41:47.050751+0000 mgr.y (mgr.34107) 181 : cluster [DBG] pgmap v122: 73 pgs: 8 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 249 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:41:49.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:49 smithi089 bash[84777]: cluster 2024-04-13T07:41:49.251333+0000 mon.a (mon.0) 363 : cluster [WRN] Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:41:49.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:49 smithi089 bash[82331]: cluster 2024-04-13T07:41:49.251333+0000 mon.a (mon.0) 363 : cluster [WRN] Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:41:49.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:49 smithi203 bash[68060]: cluster 2024-04-13T07:41:49.251333+0000 mon.a (mon.0) 363 : cluster [WRN] Health check update: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:41:50.299 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:41:50.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:50 smithi089 bash[84777]: cluster 2024-04-13T07:41:49.051990+0000 mgr.y (mgr.34107) 182 : cluster [DBG] pgmap v124: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-13T07:41:50.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:50 smithi089 bash[82331]: cluster 2024-04-13T07:41:49.051990+0000 mgr.y (mgr.34107) 182 : cluster [DBG] pgmap v124: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-13T07:41:50.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:50 smithi203 bash[68060]: cluster 2024-04-13T07:41:49.051990+0000 mgr.y (mgr.34107) 182 : cluster [DBG] pgmap v124: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2024-04-13T07:41:51.606 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:51 smithi089 bash[82331]: cluster 2024-04-13T07:41:51.305712+0000 mon.a (mon.0) 364 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-13T07:41:51.606 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:51 smithi089 bash[84777]: cluster 2024-04-13T07:41:51.305712+0000 mon.a (mon.0) 364 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-13T07:41:51.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:51 smithi203 bash[68060]: cluster 2024-04-13T07:41:51.305712+0000 mon.a (mon.0) 364 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2024-04-13T07:41:52.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:52 smithi203 bash[68060]: cluster 2024-04-13T07:41:51.052668+0000 mgr.y (mgr.34107) 183 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:52.765 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:41:52 smithi203 bash[60252]: ts=2024-04-13T07:41:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:41:52.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:52 smithi089 bash[82331]: cluster 2024-04-13T07:41:51.052668+0000 mgr.y (mgr.34107) 183 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:52.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:52 smithi089 bash[84777]: cluster 2024-04-13T07:41:51.052668+0000 mgr.y (mgr.34107) 183 : cluster [DBG] pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:54.218 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:41:54.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:54 smithi203 bash[68060]: cluster 2024-04-13T07:41:53.053207+0000 mgr.y (mgr.34107) 184 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:54.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:54 smithi203 bash[68060]: audit 2024-04-13T07:41:54.224728+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:54.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:54 smithi089 bash[82331]: cluster 2024-04-13T07:41:53.053207+0000 mgr.y (mgr.34107) 184 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:54.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:54 smithi089 bash[82331]: audit 2024-04-13T07:41:54.224728+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:54.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:54 smithi089 bash[84777]: cluster 2024-04-13T07:41:53.053207+0000 mgr.y (mgr.34107) 184 : cluster [DBG] pgmap v126: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:54.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:54 smithi089 bash[84777]: audit 2024-04-13T07:41:54.224728+0000 mon.a (mon.0) 365 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.265 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'sleep 60' 2024-04-13T07:41:55.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:54.220091+0000 mgr.y (mgr.34107) 185 : audit [DBG] from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:55.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: cephadm 2024-04-13T07:41:54.220703+0000 mgr.y (mgr.34107) 186 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:41:55.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:54.944007+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:54.956825+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:54.959414+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:54.961098+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:54.973214+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:55.068071+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:55.072004+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:55.073597+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[82331]: audit 2024-04-13T07:41:55.083997+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:54.220091+0000 mgr.y (mgr.34107) 185 : audit [DBG] from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: cephadm 2024-04-13T07:41:54.220703+0000 mgr.y (mgr.34107) 186 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:41:55.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:54.944007+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:54.956825+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:54.959414+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:55.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:54.961098+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:55.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:54.973214+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:55.068071+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:55.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:55.072004+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:55.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:55.073597+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:55.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[84777]: audit 2024-04-13T07:41:55.083997+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:54.220091+0000 mgr.y (mgr.34107) 185 : audit [DBG] from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: cephadm 2024-04-13T07:41:54.220703+0000 mgr.y (mgr.34107) 186 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:54.944007+0000 mon.a (mon.0) 366 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:54.956825+0000 mon.a (mon.0) 367 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:54.959414+0000 mon.a (mon.0) 368 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:54.961098+0000 mon.a (mon.0) 369 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:54.973214+0000 mon.a (mon.0) 370 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:55.068071+0000 mon.a (mon.0) 371 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:41:55.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:55.072004+0000 mon.a (mon.0) 372 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:41:55.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:55.073597+0000 mon.a (mon.0) 373 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:41:55.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:55 smithi203 bash[68060]: audit 2024-04-13T07:41:55.083997+0000 mon.a (mon.0) 374 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:56.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:41:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:41:55] "GET /metrics HTTP/1.1" 200 34019 "" "Prometheus/2.43.0" 2024-04-13T07:41:56.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:56 smithi203 bash[68060]: cluster 2024-04-13T07:41:55.053647+0000 mgr.y (mgr.34107) 187 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:56.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:56 smithi089 bash[84777]: cluster 2024-04-13T07:41:55.053647+0000 mgr.y (mgr.34107) 187 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:56.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:56 smithi089 bash[82331]: cluster 2024-04-13T07:41:55.053647+0000 mgr.y (mgr.34107) 187 : cluster [DBG] pgmap v127: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:58.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:58 smithi203 bash[68060]: cluster 2024-04-13T07:41:57.054469+0000 mgr.y (mgr.34107) 188 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:58 smithi203 bash[68060]: audit 2024-04-13T07:41:57.078194+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:58 smithi203 bash[68060]: audit 2024-04-13T07:41:57.083795+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:41:58 smithi203 bash[68060]: audit 2024-04-13T07:41:57.136093+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:58.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:58 smithi089 bash[84777]: cluster 2024-04-13T07:41:57.054469+0000 mgr.y (mgr.34107) 188 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:58.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:58 smithi089 bash[84777]: audit 2024-04-13T07:41:57.078194+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:58.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:58 smithi089 bash[84777]: audit 2024-04-13T07:41:57.083795+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:58.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:41:58 smithi089 bash[84777]: audit 2024-04-13T07:41:57.136093+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:58 smithi089 bash[82331]: cluster 2024-04-13T07:41:57.054469+0000 mgr.y (mgr.34107) 188 : cluster [DBG] pgmap v128: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:41:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:58 smithi089 bash[82331]: audit 2024-04-13T07:41:57.078194+0000 mon.a (mon.0) 375 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:41:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:58 smithi089 bash[82331]: audit 2024-04-13T07:41:57.083795+0000 mon.a (mon.0) 376 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:41:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:41:58 smithi089 bash[82331]: audit 2024-04-13T07:41:57.136093+0000 mon.a (mon.0) 377 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:42:00.006 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:42:00.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:00 smithi089 bash[82331]: cluster 2024-04-13T07:41:59.055519+0000 mgr.y (mgr.34107) 189 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:00.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:00 smithi089 bash[84777]: cluster 2024-04-13T07:41:59.055519+0000 mgr.y (mgr.34107) 189 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:00.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:00 smithi203 bash[68060]: cluster 2024-04-13T07:41:59.055519+0000 mgr.y (mgr.34107) 189 : cluster [DBG] pgmap v129: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:02.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:02 smithi203 bash[68060]: cluster 2024-04-13T07:42:01.056256+0000 mgr.y (mgr.34107) 190 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:02.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:42:02 smithi203 bash[60252]: ts=2024-04-13T07:42:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:42:02.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:02 smithi089 bash[82331]: cluster 2024-04-13T07:42:01.056256+0000 mgr.y (mgr.34107) 190 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:02.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:02 smithi089 bash[84777]: cluster 2024-04-13T07:42:01.056256+0000 mgr.y (mgr.34107) 190 : cluster [DBG] pgmap v130: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:04.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:04 smithi203 bash[68060]: cluster 2024-04-13T07:42:03.057237+0000 mgr.y (mgr.34107) 191 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:04.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:04 smithi089 bash[82331]: cluster 2024-04-13T07:42:03.057237+0000 mgr.y (mgr.34107) 191 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:04.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:04 smithi089 bash[84777]: cluster 2024-04-13T07:42:03.057237+0000 mgr.y (mgr.34107) 191 : cluster [DBG] pgmap v131: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:06.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:06 smithi089 bash[82331]: cluster 2024-04-13T07:42:05.057843+0000 mgr.y (mgr.34107) 192 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:06.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:42:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:42:05] "GET /metrics HTTP/1.1" 200 34008 "" "Prometheus/2.43.0" 2024-04-13T07:42:06.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:06 smithi089 bash[84777]: cluster 2024-04-13T07:42:05.057843+0000 mgr.y (mgr.34107) 192 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:06.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:06 smithi203 bash[68060]: cluster 2024-04-13T07:42:05.057843+0000 mgr.y (mgr.34107) 192 : cluster [DBG] pgmap v132: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:08.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:08 smithi203 bash[68060]: cluster 2024-04-13T07:42:07.058559+0000 mgr.y (mgr.34107) 193 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:08.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:08 smithi089 bash[84777]: cluster 2024-04-13T07:42:07.058559+0000 mgr.y (mgr.34107) 193 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:08 smithi089 bash[82331]: cluster 2024-04-13T07:42:07.058559+0000 mgr.y (mgr.34107) 193 : cluster [DBG] pgmap v133: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:10.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:10 smithi203 bash[68060]: cluster 2024-04-13T07:42:09.059576+0000 mgr.y (mgr.34107) 194 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:10.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:10 smithi089 bash[84777]: cluster 2024-04-13T07:42:09.059576+0000 mgr.y (mgr.34107) 194 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:10.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:10 smithi089 bash[82331]: cluster 2024-04-13T07:42:09.059576+0000 mgr.y (mgr.34107) 194 : cluster [DBG] pgmap v134: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:12 smithi203 bash[68060]: cluster 2024-04-13T07:42:11.060281+0000 mgr.y (mgr.34107) 195 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:12 smithi203 bash[68060]: audit 2024-04-13T07:42:12.069450+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:12.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:42:12 smithi203 bash[60252]: ts=2024-04-13T07:42:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:42:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:12 smithi089 bash[82331]: cluster 2024-04-13T07:42:11.060281+0000 mgr.y (mgr.34107) 195 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:12 smithi089 bash[82331]: audit 2024-04-13T07:42:12.069450+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:12.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:12 smithi089 bash[84777]: cluster 2024-04-13T07:42:11.060281+0000 mgr.y (mgr.34107) 195 : cluster [DBG] pgmap v135: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:12.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:12 smithi089 bash[84777]: audit 2024-04-13T07:42:12.069450+0000 mon.a (mon.0) 378 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:14.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:14 smithi203 bash[68060]: cluster 2024-04-13T07:42:13.061238+0000 mgr.y (mgr.34107) 196 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:14.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:14 smithi089 bash[82331]: cluster 2024-04-13T07:42:13.061238+0000 mgr.y (mgr.34107) 196 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:14.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:14 smithi089 bash[84777]: cluster 2024-04-13T07:42:13.061238+0000 mgr.y (mgr.34107) 196 : cluster [DBG] pgmap v136: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:16.103 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:42:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:42:15] "GET /metrics HTTP/1.1" 200 34011 "" "Prometheus/2.43.0" 2024-04-13T07:42:16.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:16 smithi203 bash[68060]: cluster 2024-04-13T07:42:15.061875+0000 mgr.y (mgr.34107) 197 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:16 smithi089 bash[84777]: cluster 2024-04-13T07:42:15.061875+0000 mgr.y (mgr.34107) 197 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:16 smithi089 bash[82331]: cluster 2024-04-13T07:42:15.061875+0000 mgr.y (mgr.34107) 197 : cluster [DBG] pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:18.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:18 smithi203 bash[68060]: cluster 2024-04-13T07:42:17.062669+0000 mgr.y (mgr.34107) 198 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:18.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:18 smithi089 bash[82331]: cluster 2024-04-13T07:42:17.062669+0000 mgr.y (mgr.34107) 198 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:18.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:18 smithi089 bash[84777]: cluster 2024-04-13T07:42:17.062669+0000 mgr.y (mgr.34107) 198 : cluster [DBG] pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:20.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:20 smithi203 bash[68060]: cluster 2024-04-13T07:42:19.063843+0000 mgr.y (mgr.34107) 199 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:20.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:20 smithi089 bash[82331]: cluster 2024-04-13T07:42:19.063843+0000 mgr.y (mgr.34107) 199 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:20.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:20 smithi089 bash[84777]: cluster 2024-04-13T07:42:19.063843+0000 mgr.y (mgr.34107) 199 : cluster [DBG] pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:22.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:22 smithi203 bash[68060]: cluster 2024-04-13T07:42:21.064700+0000 mgr.y (mgr.34107) 200 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:22.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:42:22 smithi203 bash[60252]: ts=2024-04-13T07:42:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:42:22.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:22 smithi089 bash[82331]: cluster 2024-04-13T07:42:21.064700+0000 mgr.y (mgr.34107) 200 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:22.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:22 smithi089 bash[84777]: cluster 2024-04-13T07:42:21.064700+0000 mgr.y (mgr.34107) 200 : cluster [DBG] pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:24.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:24 smithi203 bash[68060]: cluster 2024-04-13T07:42:23.065716+0000 mgr.y (mgr.34107) 201 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:24.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:24 smithi089 bash[84777]: cluster 2024-04-13T07:42:23.065716+0000 mgr.y (mgr.34107) 201 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:24.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:24 smithi089 bash[82331]: cluster 2024-04-13T07:42:23.065716+0000 mgr.y (mgr.34107) 201 : cluster [DBG] pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:26.125 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:42:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:42:25] "GET /metrics HTTP/1.1" 200 34011 "" "Prometheus/2.43.0" 2024-04-13T07:42:26.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:26 smithi203 bash[68060]: cluster 2024-04-13T07:42:25.066274+0000 mgr.y (mgr.34107) 202 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:26.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:26 smithi089 bash[84777]: cluster 2024-04-13T07:42:25.066274+0000 mgr.y (mgr.34107) 202 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:26.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:26 smithi089 bash[82331]: cluster 2024-04-13T07:42:25.066274+0000 mgr.y (mgr.34107) 202 : cluster [DBG] pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:27.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:27 smithi203 bash[68060]: audit 2024-04-13T07:42:27.070295+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:27.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:27 smithi089 bash[84777]: audit 2024-04-13T07:42:27.070295+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:27.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:27 smithi089 bash[82331]: audit 2024-04-13T07:42:27.070295+0000 mon.a (mon.0) 379 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:28.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:28 smithi203 bash[68060]: cluster 2024-04-13T07:42:27.067031+0000 mgr.y (mgr.34107) 203 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:28.555 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:28 smithi089 bash[84777]: cluster 2024-04-13T07:42:27.067031+0000 mgr.y (mgr.34107) 203 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:28.556 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:28 smithi089 bash[82331]: cluster 2024-04-13T07:42:27.067031+0000 mgr.y (mgr.34107) 203 : cluster [DBG] pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:30.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:30 smithi203 bash[68060]: cluster 2024-04-13T07:42:29.068021+0000 mgr.y (mgr.34107) 204 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:30.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:30 smithi089 bash[82331]: cluster 2024-04-13T07:42:29.068021+0000 mgr.y (mgr.34107) 204 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:30.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:30 smithi089 bash[84777]: cluster 2024-04-13T07:42:29.068021+0000 mgr.y (mgr.34107) 204 : cluster [DBG] pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:32.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:32 smithi203 bash[68060]: cluster 2024-04-13T07:42:31.068789+0000 mgr.y (mgr.34107) 205 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:32.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:42:32 smithi203 bash[60252]: ts=2024-04-13T07:42:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:42:32.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:32 smithi089 bash[82331]: cluster 2024-04-13T07:42:31.068789+0000 mgr.y (mgr.34107) 205 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:32.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:32 smithi089 bash[84777]: cluster 2024-04-13T07:42:31.068789+0000 mgr.y (mgr.34107) 205 : cluster [DBG] pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:34.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:34 smithi203 bash[68060]: cluster 2024-04-13T07:42:33.069790+0000 mgr.y (mgr.34107) 206 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:34.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:34 smithi089 bash[82331]: cluster 2024-04-13T07:42:33.069790+0000 mgr.y (mgr.34107) 206 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:34.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:34 smithi089 bash[84777]: cluster 2024-04-13T07:42:33.069790+0000 mgr.y (mgr.34107) 206 : cluster [DBG] pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:36.154 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:42:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:42:35] "GET /metrics HTTP/1.1" 200 34014 "" "Prometheus/2.43.0" 2024-04-13T07:42:36.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:36 smithi203 bash[68060]: cluster 2024-04-13T07:42:35.070545+0000 mgr.y (mgr.34107) 207 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:36.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:36 smithi089 bash[82331]: cluster 2024-04-13T07:42:35.070545+0000 mgr.y (mgr.34107) 207 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:36.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:36 smithi089 bash[84777]: cluster 2024-04-13T07:42:35.070545+0000 mgr.y (mgr.34107) 207 : cluster [DBG] pgmap v147: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:38.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:38 smithi203 bash[68060]: cluster 2024-04-13T07:42:37.071341+0000 mgr.y (mgr.34107) 208 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:38.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:38 smithi089 bash[82331]: cluster 2024-04-13T07:42:37.071341+0000 mgr.y (mgr.34107) 208 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:38.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:38 smithi089 bash[84777]: cluster 2024-04-13T07:42:37.071341+0000 mgr.y (mgr.34107) 208 : cluster [DBG] pgmap v148: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:40.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:40 smithi203 bash[68060]: cluster 2024-04-13T07:42:39.072363+0000 mgr.y (mgr.34107) 209 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:40.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:40 smithi089 bash[82331]: cluster 2024-04-13T07:42:39.072363+0000 mgr.y (mgr.34107) 209 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:40.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:40 smithi089 bash[84777]: cluster 2024-04-13T07:42:39.072363+0000 mgr.y (mgr.34107) 209 : cluster [DBG] pgmap v149: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:42.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:42:42 smithi203 bash[60252]: ts=2024-04-13T07:42:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:42:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:42 smithi203 bash[68060]: cluster 2024-04-13T07:42:41.073115+0000 mgr.y (mgr.34107) 210 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:42 smithi203 bash[68060]: audit 2024-04-13T07:42:42.070400+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:42.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:42 smithi089 bash[82331]: cluster 2024-04-13T07:42:41.073115+0000 mgr.y (mgr.34107) 210 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:42.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:42 smithi089 bash[82331]: audit 2024-04-13T07:42:42.070400+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:42.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:42 smithi089 bash[84777]: cluster 2024-04-13T07:42:41.073115+0000 mgr.y (mgr.34107) 210 : cluster [DBG] pgmap v150: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:42.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:42 smithi089 bash[84777]: audit 2024-04-13T07:42:42.070400+0000 mon.a (mon.0) 380 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:44 smithi203 bash[68060]: cluster 2024-04-13T07:42:43.074111+0000 mgr.y (mgr.34107) 211 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:44.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:44 smithi089 bash[84777]: cluster 2024-04-13T07:42:43.074111+0000 mgr.y (mgr.34107) 211 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:44.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:44 smithi089 bash[82331]: cluster 2024-04-13T07:42:43.074111+0000 mgr.y (mgr.34107) 211 : cluster [DBG] pgmap v151: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:46.175 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:42:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:42:45] "GET /metrics HTTP/1.1" 200 34015 "" "Prometheus/2.43.0" 2024-04-13T07:42:46.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:46 smithi203 bash[68060]: cluster 2024-04-13T07:42:45.074720+0000 mgr.y (mgr.34107) 212 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:46.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:46 smithi089 bash[82331]: cluster 2024-04-13T07:42:45.074720+0000 mgr.y (mgr.34107) 212 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:46.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:46 smithi089 bash[84777]: cluster 2024-04-13T07:42:45.074720+0000 mgr.y (mgr.34107) 212 : cluster [DBG] pgmap v152: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:48.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:48 smithi203 bash[68060]: cluster 2024-04-13T07:42:47.075532+0000 mgr.y (mgr.34107) 213 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:48.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:48 smithi089 bash[82331]: cluster 2024-04-13T07:42:47.075532+0000 mgr.y (mgr.34107) 213 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:48.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:48 smithi089 bash[84777]: cluster 2024-04-13T07:42:47.075532+0000 mgr.y (mgr.34107) 213 : cluster [DBG] pgmap v153: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:50.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:50 smithi203 bash[68060]: cluster 2024-04-13T07:42:49.076632+0000 mgr.y (mgr.34107) 214 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:50.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:50 smithi089 bash[84777]: cluster 2024-04-13T07:42:49.076632+0000 mgr.y (mgr.34107) 214 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:50.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:50 smithi089 bash[82331]: cluster 2024-04-13T07:42:49.076632+0000 mgr.y (mgr.34107) 214 : cluster [DBG] pgmap v154: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:52.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:52 smithi203 bash[68060]: cluster 2024-04-13T07:42:51.077375+0000 mgr.y (mgr.34107) 215 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:52.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:42:52 smithi203 bash[60252]: ts=2024-04-13T07:42:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:42:52.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:52 smithi089 bash[82331]: cluster 2024-04-13T07:42:51.077375+0000 mgr.y (mgr.34107) 215 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:52.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:52 smithi089 bash[84777]: cluster 2024-04-13T07:42:51.077375+0000 mgr.y (mgr.34107) 215 : cluster [DBG] pgmap v155: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:54.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:54 smithi203 bash[68060]: cluster 2024-04-13T07:42:53.078378+0000 mgr.y (mgr.34107) 216 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:54.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:54 smithi089 bash[82331]: cluster 2024-04-13T07:42:53.078378+0000 mgr.y (mgr.34107) 216 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:54.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:54 smithi089 bash[84777]: cluster 2024-04-13T07:42:53.078378+0000 mgr.y (mgr.34107) 216 : cluster [DBG] pgmap v156: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:55.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:55 smithi203 bash[68060]: audit 2024-04-13T07:42:55.164986+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:42:55.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:55 smithi089 bash[84777]: audit 2024-04-13T07:42:55.164986+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:42:55.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:55 smithi089 bash[82331]: audit 2024-04-13T07:42:55.164986+0000 mon.a (mon.0) 381 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:42:56.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:42:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:42:55] "GET /metrics HTTP/1.1" 200 34015 "" "Prometheus/2.43.0" 2024-04-13T07:42:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:56 smithi203 bash[68060]: cluster 2024-04-13T07:42:55.079005+0000 mgr.y (mgr.34107) 217 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:56 smithi203 bash[68060]: audit 2024-04-13T07:42:55.744713+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:42:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:56 smithi203 bash[68060]: audit 2024-04-13T07:42:55.756468+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:42:57.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:56 smithi089 bash[84777]: cluster 2024-04-13T07:42:55.079005+0000 mgr.y (mgr.34107) 217 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:57.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:56 smithi089 bash[84777]: audit 2024-04-13T07:42:55.744713+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:42:57.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:56 smithi089 bash[84777]: audit 2024-04-13T07:42:55.756468+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:42:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:56 smithi089 bash[82331]: cluster 2024-04-13T07:42:55.079005+0000 mgr.y (mgr.34107) 217 : cluster [DBG] pgmap v157: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:56 smithi089 bash[82331]: audit 2024-04-13T07:42:55.744713+0000 mon.a (mon.0) 382 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:42:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:56 smithi089 bash[82331]: audit 2024-04-13T07:42:55.756468+0000 mon.a (mon.0) 383 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:42:58.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:57 smithi203 bash[68060]: audit 2024-04-13T07:42:57.071162+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:58.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:57 smithi089 bash[82331]: audit 2024-04-13T07:42:57.071162+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:58.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:57 smithi089 bash[84777]: audit 2024-04-13T07:42:57.071162+0000 mon.a (mon.0) 384 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:42:59.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:42:58 smithi203 bash[68060]: cluster 2024-04-13T07:42:57.083424+0000 mgr.y (mgr.34107) 218 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:59.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:42:58 smithi089 bash[82331]: cluster 2024-04-13T07:42:57.083424+0000 mgr.y (mgr.34107) 218 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:42:59.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:42:58 smithi089 bash[84777]: cluster 2024-04-13T07:42:57.083424+0000 mgr.y (mgr.34107) 218 : cluster [DBG] pgmap v158: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:01.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:00 smithi203 bash[68060]: cluster 2024-04-13T07:42:59.084377+0000 mgr.y (mgr.34107) 219 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:01.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:00 smithi089 bash[84777]: cluster 2024-04-13T07:42:59.084377+0000 mgr.y (mgr.34107) 219 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:01.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:00 smithi089 bash[82331]: cluster 2024-04-13T07:42:59.084377+0000 mgr.y (mgr.34107) 219 : cluster [DBG] pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:02.264 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:02 smithi203 bash[68060]: audit 2024-04-13T07:43:01.019937+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:02 smithi203 bash[68060]: audit 2024-04-13T07:43:01.032557+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:02 smithi203 bash[68060]: cluster 2024-04-13T07:43:01.084873+0000 mgr.y (mgr.34107) 220 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:02.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:02 smithi203 bash[68060]: audit 2024-04-13T07:43:01.718474+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:02 smithi203 bash[68060]: audit 2024-04-13T07:43:01.730172+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[82331]: audit 2024-04-13T07:43:01.019937+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[82331]: audit 2024-04-13T07:43:01.032557+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[82331]: cluster 2024-04-13T07:43:01.084873+0000 mgr.y (mgr.34107) 220 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:02.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[82331]: audit 2024-04-13T07:43:01.718474+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.272 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[82331]: audit 2024-04-13T07:43:01.730172+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[84777]: audit 2024-04-13T07:43:01.019937+0000 mon.a (mon.0) 385 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.272 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[84777]: audit 2024-04-13T07:43:01.032557+0000 mon.a (mon.0) 386 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.273 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[84777]: cluster 2024-04-13T07:43:01.084873+0000 mgr.y (mgr.34107) 220 : cluster [DBG] pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:02.273 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[84777]: audit 2024-04-13T07:43:01.718474+0000 mon.a (mon.0) 387 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.273 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:02 smithi089 bash[84777]: audit 2024-04-13T07:43:01.730172+0000 mon.a (mon.0) 388 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:02.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:43:02 smithi203 bash[60252]: ts=2024-04-13T07:43:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:43:02.889 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch upgrade resume' 2024-04-13T07:43:03.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:03 smithi089 bash[82331]: audit 2024-04-13T07:43:02.363998+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:03.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:03 smithi089 bash[82331]: audit 2024-04-13T07:43:02.365278+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:03.342 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:03 smithi089 bash[82331]: audit 2024-04-13T07:43:02.371908+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:03.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:03 smithi089 bash[84777]: audit 2024-04-13T07:43:02.363998+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:03.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:03 smithi089 bash[84777]: audit 2024-04-13T07:43:02.365278+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:03.343 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:03 smithi089 bash[84777]: audit 2024-04-13T07:43:02.371908+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:03.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:03 smithi203 bash[68060]: audit 2024-04-13T07:43:02.363998+0000 mon.a (mon.0) 389 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:03.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:03 smithi203 bash[68060]: audit 2024-04-13T07:43:02.365278+0000 mon.a (mon.0) 390 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:03.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:03 smithi203 bash[68060]: audit 2024-04-13T07:43:02.371908+0000 mon.a (mon.0) 391 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:04.297 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:04 smithi089 bash[82331]: cluster 2024-04-13T07:43:03.085841+0000 mgr.y (mgr.34107) 221 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:04.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:04 smithi089 bash[84777]: cluster 2024-04-13T07:43:03.085841+0000 mgr.y (mgr.34107) 221 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:04.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:04 smithi203 bash[68060]: cluster 2024-04-13T07:43:03.085841+0000 mgr.y (mgr.34107) 221 : cluster [DBG] pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:06.135 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:43:05] "GET /metrics HTTP/1.1" 200 34009 "" "Prometheus/2.43.0" 2024-04-13T07:43:06.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:06 smithi203 bash[68060]: cluster 2024-04-13T07:43:05.086296+0000 mgr.y (mgr.34107) 222 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:06.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:06 smithi089 bash[82331]: cluster 2024-04-13T07:43:05.086296+0000 mgr.y (mgr.34107) 222 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:06.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:06 smithi089 bash[84777]: cluster 2024-04-13T07:43:05.086296+0000 mgr.y (mgr.34107) 222 : cluster [DBG] pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:07.626 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:43:08.389 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:08 smithi089 bash[84777]: cluster 2024-04-13T07:43:07.086952+0000 mgr.y (mgr.34107) 223 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:08.389 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:08 smithi089 bash[82331]: cluster 2024-04-13T07:43:07.086952+0000 mgr.y (mgr.34107) 223 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:08.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:08 smithi203 bash[68060]: cluster 2024-04-13T07:43:07.086952+0000 mgr.y (mgr.34107) 223 : cluster [DBG] pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:10.237 INFO:teuthology.orchestra.run.smithi089.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:43:10.436 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:10 smithi089 bash[82331]: cluster 2024-04-13T07:43:09.087955+0000 mgr.y (mgr.34107) 224 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:10.437 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:10 smithi089 bash[84777]: cluster 2024-04-13T07:43:09.087955+0000 mgr.y (mgr.34107) 224 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:10.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:10 smithi203 bash[68060]: cluster 2024-04-13T07:43:09.087955+0000 mgr.y (mgr.34107) 224 : cluster [DBG] pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:11.114 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'echo wait for half of osds to upgrade' 2024-04-13T07:43:11.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.236008+0000 mgr.y (mgr.34107) 225 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:43:11.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: cephadm 2024-04-13T07:43:10.236367+0000 mgr.y (mgr.34107) 226 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:43:11.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.244117+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:11.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.281450+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:11.438 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.283052+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.283736+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.290896+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.350613+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.352766+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.354396+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.355529+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.357106+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: audit 2024-04-13T07:43:10.357253+0000 mgr.y (mgr.34107) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[82331]: cephadm 2024-04-13T07:43:10.358016+0000 mgr.y (mgr.34107) 228 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.236008+0000 mgr.y (mgr.34107) 225 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: cephadm 2024-04-13T07:43:10.236367+0000 mgr.y (mgr.34107) 226 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:43:11.439 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.244117+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.281450+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.283052+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.283736+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.290896+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.350613+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.352766+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.354396+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.355529+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.357106+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: audit 2024-04-13T07:43:10.357253+0000 mgr.y (mgr.34107) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-13T07:43:11.440 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:11 smithi089 bash[84777]: cephadm 2024-04-13T07:43:10.358016+0000 mgr.y (mgr.34107) 228 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.236008+0000 mgr.y (mgr.34107) 225 : audit [DBG] from='client.44150 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: cephadm 2024-04-13T07:43:10.236367+0000 mgr.y (mgr.34107) 226 : cephadm [INF] Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.244117+0000 mon.a (mon.0) 392 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.281450+0000 mon.a (mon.0) 393 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.283052+0000 mon.a (mon.0) 394 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.283736+0000 mon.a (mon.0) 395 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.290896+0000 mon.a (mon.0) 396 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.350613+0000 mon.a (mon.0) 397 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.352766+0000 mon.a (mon.0) 398 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.354396+0000 mon.a (mon.0) 399 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.355529+0000 mon.a (mon.0) 400 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.357106+0000 mon.a (mon.0) 401 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: audit 2024-04-13T07:43:10.357253+0000 mgr.y (mgr.34107) 227 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2024-04-13T07:43:11.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:11 smithi203 bash[68060]: cephadm 2024-04-13T07:43:10.358016+0000 mgr.y (mgr.34107) 228 : cephadm [INF] Upgrade: osd.1 is safe to restart 2024-04-13T07:43:12.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:12 smithi089 bash[84777]: cluster 2024-04-13T07:43:11.088871+0000 mgr.y (mgr.34107) 229 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:12.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:12 smithi089 bash[84777]: audit 2024-04-13T07:43:12.071646+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:12 smithi089 bash[82331]: cluster 2024-04-13T07:43:11.088871+0000 mgr.y (mgr.34107) 229 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:12 smithi089 bash[82331]: audit 2024-04-13T07:43:12.071646+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:12.765 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:43:12 smithi203 bash[60252]: ts=2024-04-13T07:43:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:43:12.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:12 smithi203 bash[68060]: cluster 2024-04-13T07:43:11.088871+0000 mgr.y (mgr.34107) 229 : cluster [DBG] pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:12.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:12 smithi203 bash[68060]: audit 2024-04-13T07:43:12.071646+0000 mon.a (mon.0) 402 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:13.639 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[84777]: cephadm 2024-04-13T07:43:12.326751+0000 mgr.y (mgr.34107) 230 : cephadm [INF] Upgrade: Updating osd.1 2024-04-13T07:43:13.639 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[84777]: audit 2024-04-13T07:43:12.333545+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:13.639 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[84777]: audit 2024-04-13T07:43:12.338149+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:43:13.639 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[84777]: audit 2024-04-13T07:43:12.339570+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:13.639 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[84777]: cephadm 2024-04-13T07:43:12.344098+0000 mgr.y (mgr.34107) 231 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-04-13T07:43:13.640 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[82331]: cephadm 2024-04-13T07:43:12.326751+0000 mgr.y (mgr.34107) 230 : cephadm [INF] Upgrade: Updating osd.1 2024-04-13T07:43:13.640 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[82331]: audit 2024-04-13T07:43:12.333545+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:13.640 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[82331]: audit 2024-04-13T07:43:12.338149+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:43:13.640 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[82331]: audit 2024-04-13T07:43:12.339570+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:13.640 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:13 smithi089 bash[82331]: cephadm 2024-04-13T07:43:12.344098+0000 mgr.y (mgr.34107) 231 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-04-13T07:43:13.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:13 smithi203 bash[68060]: cephadm 2024-04-13T07:43:12.326751+0000 mgr.y (mgr.34107) 230 : cephadm [INF] Upgrade: Updating osd.1 2024-04-13T07:43:13.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:13 smithi203 bash[68060]: audit 2024-04-13T07:43:12.333545+0000 mon.a (mon.0) 403 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:13.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:13 smithi203 bash[68060]: audit 2024-04-13T07:43:12.338149+0000 mon.a (mon.0) 404 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2024-04-13T07:43:13.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:13 smithi203 bash[68060]: audit 2024-04-13T07:43:12.339570+0000 mon.a (mon.0) 405 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:13.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:13 smithi203 bash[68060]: cephadm 2024-04-13T07:43:12.344098+0000 mgr.y (mgr.34107) 231 : cephadm [INF] Deploying daemon osd.1 on smithi089 2024-04-13T07:43:14.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:14 smithi203 bash[68060]: cluster 2024-04-13T07:43:13.089919+0000 mgr.y (mgr.34107) 232 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:14.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:14 smithi089 bash[82331]: cluster 2024-04-13T07:43:13.089919+0000 mgr.y (mgr.34107) 232 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:14.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:14 smithi089 bash[84777]: cluster 2024-04-13T07:43:13.089919+0000 mgr.y (mgr.34107) 232 : cluster [DBG] pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:16.235 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:43:15] "GET /metrics HTTP/1.1" 200 34014 "" "Prometheus/2.43.0" 2024-04-13T07:43:16.400 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:43:16.547 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.548 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.548 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.548 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.549 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.549 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:16 smithi089 bash[82331]: cluster 2024-04-13T07:43:15.090342+0000 mgr.y (mgr.34107) 233 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:16.550 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.550 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: Stopping Ceph osd.1 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:43:16.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:16 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:16.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:16 smithi089 bash[84777]: cluster 2024-04-13T07:43:15.090342+0000 mgr.y (mgr.34107) 233 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:16.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:16 smithi203 bash[68060]: cluster 2024-04-13T07:43:15.090342+0000 mgr.y (mgr.34107) 233 : cluster [DBG] pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:17.047 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:16 smithi089 bash[40301]: debug 2024-04-13T07:43:16.606+0000 7fc3aea9c700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:43:17.048 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:16 smithi089 bash[40301]: debug 2024-04-13T07:43:16.606+0000 7fc3aea9c700 -1 osd.1 97 *** Got signal Terminated *** 2024-04-13T07:43:17.048 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:16 smithi089 bash[40301]: debug 2024-04-13T07:43:16.606+0000 7fc3aea9c700 -1 osd.1 97 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:43:17.719 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:17 smithi089 bash[84777]: cluster 2024-04-13T07:43:16.610950+0000 mon.a (mon.0) 406 : cluster [INF] osd.1 marked itself down and dead 2024-04-13T07:43:17.719 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:17 smithi089 bash[82331]: debug 2024-04-13T07:43:17.342+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:17.719 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:17 smithi089 bash[82331]: cluster 2024-04-13T07:43:16.610950+0000 mon.a (mon.0) 406 : cluster [INF] osd.1 marked itself down and dead 2024-04-13T07:43:17.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:17 smithi203 bash[68060]: cluster 2024-04-13T07:43:16.610950+0000 mon.a (mon.0) 406 : cluster [INF] osd.1 marked itself down and dead 2024-04-13T07:43:18.547 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[95750]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-1 2024-04-13T07:43:18.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[82331]: cluster 2024-04-13T07:43:17.091051+0000 mgr.y (mgr.34107) 234 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:18.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[82331]: cluster 2024-04-13T07:43:17.343836+0000 mon.a (mon.0) 407 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:43:18.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[82331]: cluster 2024-04-13T07:43:17.343893+0000 mon.a (mon.0) 408 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:18.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[82331]: cluster 2024-04-13T07:43:17.358937+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e98: 8 total, 7 up, 8 in 2024-04-13T07:43:18.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[84777]: cluster 2024-04-13T07:43:17.091051+0000 mgr.y (mgr.34107) 234 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:18.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[84777]: cluster 2024-04-13T07:43:17.343836+0000 mon.a (mon.0) 407 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:43:18.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[84777]: cluster 2024-04-13T07:43:17.343893+0000 mon.a (mon.0) 408 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:18.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:18 smithi089 bash[84777]: cluster 2024-04-13T07:43:17.358937+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e98: 8 total, 7 up, 8 in 2024-04-13T07:43:18.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:18 smithi203 bash[68060]: cluster 2024-04-13T07:43:17.091051+0000 mgr.y (mgr.34107) 234 : cluster [DBG] pgmap v168: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:18.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:18 smithi203 bash[68060]: cluster 2024-04-13T07:43:17.343836+0000 mon.a (mon.0) 407 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:43:18.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:18 smithi203 bash[68060]: cluster 2024-04-13T07:43:17.343893+0000 mon.a (mon.0) 408 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:18.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:18 smithi203 bash[68060]: cluster 2024-04-13T07:43:17.358937+0000 mon.a (mon.0) 409 : cluster [DBG] osdmap e98: 8 total, 7 up, 8 in 2024-04-13T07:43:19.297 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:19 smithi089 bash[95898]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-1 2024-04-13T07:43:19.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:19 smithi203 bash[68060]: cluster 2024-04-13T07:43:18.360268+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e99: 8 total, 7 up, 8 in 2024-04-13T07:43:19.797 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:19 smithi089 bash[82331]: cluster 2024-04-13T07:43:18.360268+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e99: 8 total, 7 up, 8 in 2024-04-13T07:43:19.797 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:19 smithi089 bash[84777]: cluster 2024-04-13T07:43:18.360268+0000 mon.a (mon.0) 410 : cluster [DBG] osdmap e99: 8 total, 7 up, 8 in 2024-04-13T07:43:19.963 INFO:teuthology.orchestra.run.smithi089.stdout:wait for half of osds to upgrade 2024-04-13T07:43:20.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:20 smithi089 bash[82331]: cluster 2024-04-13T07:43:19.091672+0000 mgr.y (mgr.34107) 235 : cluster [DBG] pgmap v171: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:20.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:20 smithi089 bash[82331]: cluster 2024-04-13T07:43:19.354723+0000 mon.a (mon.0) 411 : cluster [WRN] Health check failed: Reduced data availability: 7 pgs inactive, 21 pgs peering (PG_AVAILABILITY) 2024-04-13T07:43:20.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:20 smithi089 bash[84777]: cluster 2024-04-13T07:43:19.091672+0000 mgr.y (mgr.34107) 235 : cluster [DBG] pgmap v171: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:20.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:20 smithi089 bash[84777]: cluster 2024-04-13T07:43:19.354723+0000 mon.a (mon.0) 411 : cluster [WRN] Health check failed: Reduced data availability: 7 pgs inactive, 21 pgs peering (PG_AVAILABILITY) 2024-04-13T07:43:20.673 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2024-04-13T07:43:20.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:20 smithi203 bash[68060]: cluster 2024-04-13T07:43:19.091672+0000 mgr.y (mgr.34107) 235 : cluster [DBG] pgmap v171: 73 pgs: 28 peering, 1 stale+active+clean, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:20.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:20 smithi203 bash[68060]: cluster 2024-04-13T07:43:19.354723+0000 mon.a (mon.0) 411 : cluster [WRN] Health check failed: Reduced data availability: 7 pgs inactive, 21 pgs peering (PG_AVAILABILITY) 2024-04-13T07:43:21.638 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:21.638 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:21.639 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:21.639 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:21.639 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.1.service: Deactivated successfully. 2024-04-13T07:43:21.639 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: Stopped Ceph osd.1 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:43:21.639 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:21.640 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:21.640 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:21.640 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:21.641 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:22.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:21 smithi203 bash[68060]: cluster 2024-04-13T07:43:21.621233+0000 mon.a (mon.0) 412 : cluster [WRN] Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:22.039 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:21 smithi089 bash[82331]: cluster 2024-04-13T07:43:21.621233+0000 mon.a (mon.0) 412 : cluster [WRN] Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:22.039 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:21 smithi089 systemd[1]: Started Ceph osd.1 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:43:22.039 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:21 smithi089 bash[84777]: cluster 2024-04-13T07:43:21.621233+0000 mon.a (mon.0) 412 : cluster [WRN] Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:22.692 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:43:22 smithi203 bash[60252]: ts=2024-04-13T07:43:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:43:22.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:22 smithi089 bash[84777]: cluster 2024-04-13T07:43:21.092786+0000 mgr.y (mgr.34107) 236 : cluster [DBG] pgmap v172: 73 pgs: 2 active+undersized+degraded, 28 peering, 2 active+undersized, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:22.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:22 smithi089 bash[84777]: audit 2024-04-13T07:43:21.698714+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:22.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:22 smithi089 bash[84777]: audit 2024-04-13T07:43:21.705781+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:22.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:22 smithi089 bash[84777]: audit 2024-04-13T07:43:21.727820+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:22 smithi089 bash[82331]: cluster 2024-04-13T07:43:21.092786+0000 mgr.y (mgr.34107) 236 : cluster [DBG] pgmap v172: 73 pgs: 2 active+undersized+degraded, 28 peering, 2 active+undersized, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:22 smithi089 bash[82331]: audit 2024-04-13T07:43:21.698714+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:22 smithi089 bash[82331]: audit 2024-04-13T07:43:21.705781+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:22 smithi089 bash[82331]: audit 2024-04-13T07:43:21.727820+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:23.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:22 smithi203 bash[68060]: cluster 2024-04-13T07:43:21.092786+0000 mgr.y (mgr.34107) 236 : cluster [DBG] pgmap v172: 73 pgs: 2 active+undersized+degraded, 28 peering, 2 active+undersized, 41 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:23.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:22 smithi203 bash[68060]: audit 2024-04-13T07:43:21.698714+0000 mon.a (mon.0) 413 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:23.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:22 smithi203 bash[68060]: audit 2024-04-13T07:43:21.705781+0000 mon.a (mon.0) 414 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:23.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:22 smithi203 bash[68060]: audit 2024-04-13T07:43:21.727820+0000 mon.a (mon.0) 415 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:23.297 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[96149]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-13T07:43:23.297 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[96149]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-1 --no-mon-config --dev /dev/mapper/vg_nvme-lv_3 2024-04-13T07:43:23.297 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[96149]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_3 2024-04-13T07:43:23.298 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[96149]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2024-04-13T07:43:23.298 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[96149]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_3 /var/lib/ceph/osd/ceph-1/block 2024-04-13T07:43:23.298 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[96149]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2024-04-13T07:43:23.298 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[96149]: --> ceph-volume raw activate successful for osd ID: 1 2024-04-13T07:43:24.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:23 smithi203 bash[68060]: cluster 2024-04-13T07:43:23.698893+0000 mon.a (mon.0) 416 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 21 pgs peering) 2024-04-13T07:43:24.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[82331]: cluster 2024-04-13T07:43:23.698893+0000 mon.a (mon.0) 416 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 21 pgs peering) 2024-04-13T07:43:24.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:23 smithi089 bash[84777]: cluster 2024-04-13T07:43:23.698893+0000 mon.a (mon.0) 416 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 7 pgs inactive, 21 pgs peering) 2024-04-13T07:43:24.746 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:24 smithi089 bash[96407]: debug 2024-04-13T07:43:24.534+0000 7f7c2d8d8700 1 -- 172.21.15.89:0/1509974115 <== mon.2 v2:172.21.15.89:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x56343ba11680 con 0x56343ba4f000 2024-04-13T07:43:25.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:24 smithi203 bash[68060]: cluster 2024-04-13T07:43:23.093510+0000 mgr.y (mgr.34107) 237 : cluster [DBG] pgmap v173: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:25.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:24 smithi089 bash[82331]: cluster 2024-04-13T07:43:23.093510+0000 mgr.y (mgr.34107) 237 : cluster [DBG] pgmap v173: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:25.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:24 smithi089 bash[84777]: cluster 2024-04-13T07:43:23.093510+0000 mgr.y (mgr.34107) 237 : cluster [DBG] pgmap v173: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:25.547 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:25 smithi089 bash[96407]: debug 2024-04-13T07:43:25.098+0000 7f7c36a707c0 -1 Falling back to public interface 2024-04-13T07:43:26.297 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:43:25] "GET /metrics HTTP/1.1" 200 34014 "" "Prometheus/2.43.0" 2024-04-13T07:43:26.393 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:43:27.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:26 smithi203 bash[68060]: cluster 2024-04-13T07:43:25.094242+0000 mgr.y (mgr.34107) 238 : cluster [DBG] pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:27.047 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:26 smithi089 bash[84777]: cluster 2024-04-13T07:43:25.094242+0000 mgr.y (mgr.34107) 238 : cluster [DBG] pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:27.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:26 smithi089 bash[82331]: cluster 2024-04-13T07:43:25.094242+0000 mgr.y (mgr.34107) 238 : cluster [DBG] pgmap v174: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:28.297 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[96407]: debug 2024-04-13T07:43:28.174+0000 7f7c36a707c0 -1 osd.1 97 log_to_monitors true 2024-04-13T07:43:28.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[84777]: audit 2024-04-13T07:43:27.073818+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:28.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[84777]: audit 2024-04-13T07:43:27.074663+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:28.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[84777]: cluster 2024-04-13T07:43:27.094841+0000 mgr.y (mgr.34107) 239 : cluster [DBG] pgmap v175: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:28.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[84777]: audit 2024-04-13T07:43:27.109105+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:28.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[84777]: audit 2024-04-13T07:43:27.117853+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:28.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[82331]: audit 2024-04-13T07:43:27.073818+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:28.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[82331]: audit 2024-04-13T07:43:27.074663+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:28.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[82331]: cluster 2024-04-13T07:43:27.094841+0000 mgr.y (mgr.34107) 239 : cluster [DBG] pgmap v175: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:28.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[82331]: audit 2024-04-13T07:43:27.109105+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:28.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:28 smithi089 bash[82331]: audit 2024-04-13T07:43:27.117853+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:28.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:28 smithi203 bash[68060]: audit 2024-04-13T07:43:27.073818+0000 mon.a (mon.0) 417 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:28.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:28 smithi203 bash[68060]: audit 2024-04-13T07:43:27.074663+0000 mon.a (mon.0) 418 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:28.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:28 smithi203 bash[68060]: cluster 2024-04-13T07:43:27.094841+0000 mgr.y (mgr.34107) 239 : cluster [DBG] pgmap v175: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:28.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:28 smithi203 bash[68060]: audit 2024-04-13T07:43:27.109105+0000 mon.a (mon.0) 419 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:28.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:28 smithi203 bash[68060]: audit 2024-04-13T07:43:27.117853+0000 mon.a (mon.0) 420 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:29.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:29 smithi203 bash[68060]: audit 2024-04-13T07:43:28.104358+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:29.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:29 smithi203 bash[68060]: audit 2024-04-13T07:43:28.110413+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:29.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:29 smithi203 bash[68060]: audit 2024-04-13T07:43:28.180408+0000 mon.a (mon.0) 423 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:43:29.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:29 smithi089 bash[82331]: audit 2024-04-13T07:43:28.104358+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:29.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:29 smithi089 bash[82331]: audit 2024-04-13T07:43:28.110413+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:29.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:29 smithi089 bash[82331]: audit 2024-04-13T07:43:28.180408+0000 mon.a (mon.0) 423 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:43:29.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:29 smithi089 bash[84777]: audit 2024-04-13T07:43:28.104358+0000 mon.a (mon.0) 421 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:29.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:29 smithi089 bash[84777]: audit 2024-04-13T07:43:28.110413+0000 mon.a (mon.0) 422 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:29.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:29 smithi089 bash[84777]: audit 2024-04-13T07:43:28.180408+0000 mon.a (mon.0) 423 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2024-04-13T07:43:30.047 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:29 smithi089 bash[96407]: debug 2024-04-13T07:43:29.614+0000 7f7c23ec0700 -1 osd.1 97 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-13T07:43:30.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:30 smithi203 bash[68060]: cluster 2024-04-13T07:43:29.095865+0000 mgr.y (mgr.34107) 240 : cluster [DBG] pgmap v176: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:30.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:30 smithi203 bash[68060]: audit 2024-04-13T07:43:29.112899+0000 mon.a (mon.0) 424 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-13T07:43:30.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:30 smithi203 bash[68060]: cluster 2024-04-13T07:43:29.120322+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e100: 8 total, 7 up, 8 in 2024-04-13T07:43:30.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:30 smithi203 bash[68060]: audit 2024-04-13T07:43:29.121566+0000 mon.a (mon.0) 426 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:43:30.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:30 smithi203 bash[68060]: audit 2024-04-13T07:43:29.129039+0000 mon.a (mon.0) 427 : audit [DBG] from='client.? 172.21.15.89:0/10485222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:30.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:30 smithi203 bash[68060]: audit 2024-04-13T07:43:29.597822+0000 mon.a (mon.0) 428 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' 2024-04-13T07:43:30.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[82331]: cluster 2024-04-13T07:43:29.095865+0000 mgr.y (mgr.34107) 240 : cluster [DBG] pgmap v176: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:30.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[82331]: audit 2024-04-13T07:43:29.112899+0000 mon.a (mon.0) 424 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-13T07:43:30.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[82331]: cluster 2024-04-13T07:43:29.120322+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e100: 8 total, 7 up, 8 in 2024-04-13T07:43:30.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[82331]: audit 2024-04-13T07:43:29.121566+0000 mon.a (mon.0) 426 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:43:30.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[82331]: audit 2024-04-13T07:43:29.129039+0000 mon.a (mon.0) 427 : audit [DBG] from='client.? 172.21.15.89:0/10485222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:30.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[82331]: audit 2024-04-13T07:43:29.597822+0000 mon.a (mon.0) 428 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' 2024-04-13T07:43:30.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[84777]: cluster 2024-04-13T07:43:29.095865+0000 mgr.y (mgr.34107) 240 : cluster [DBG] pgmap v176: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:30.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[84777]: audit 2024-04-13T07:43:29.112899+0000 mon.a (mon.0) 424 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2024-04-13T07:43:30.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[84777]: cluster 2024-04-13T07:43:29.120322+0000 mon.a (mon.0) 425 : cluster [DBG] osdmap e100: 8 total, 7 up, 8 in 2024-04-13T07:43:30.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[84777]: audit 2024-04-13T07:43:29.121566+0000 mon.a (mon.0) 426 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:43:30.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[84777]: audit 2024-04-13T07:43:29.129039+0000 mon.a (mon.0) 427 : audit [DBG] from='client.? 172.21.15.89:0/10485222' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:30.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[84777]: audit 2024-04-13T07:43:29.597822+0000 mon.a (mon.0) 428 : audit [INF] from='osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631]' entity='osd.1' 2024-04-13T07:43:31.047 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:30 smithi089 bash[82331]: debug 2024-04-13T07:43:30.594+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:31.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:31 smithi203 bash[68060]: cluster 2024-04-13T07:43:30.597069+0000 mon.a (mon.0) 429 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:31 smithi203 bash[68060]: cluster 2024-04-13T07:43:30.597128+0000 mon.a (mon.0) 430 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:43:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:31 smithi203 bash[68060]: cluster 2024-04-13T07:43:30.608110+0000 mon.a (mon.0) 431 : cluster [INF] osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631] boot 2024-04-13T07:43:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:31 smithi203 bash[68060]: cluster 2024-04-13T07:43:30.608296+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-04-13T07:43:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:31 smithi203 bash[68060]: audit 2024-04-13T07:43:30.609128+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:43:31.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:31 smithi203 bash[68060]: cluster 2024-04-13T07:43:30.621636+0000 mon.a (mon.0) 434 : cluster [WRN] Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:31.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[82331]: cluster 2024-04-13T07:43:30.597069+0000 mon.a (mon.0) 429 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:31.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[82331]: cluster 2024-04-13T07:43:30.597128+0000 mon.a (mon.0) 430 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:43:31.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[82331]: cluster 2024-04-13T07:43:30.608110+0000 mon.a (mon.0) 431 : cluster [INF] osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631] boot 2024-04-13T07:43:31.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[82331]: cluster 2024-04-13T07:43:30.608296+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-04-13T07:43:31.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[82331]: audit 2024-04-13T07:43:30.609128+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:43:31.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[82331]: cluster 2024-04-13T07:43:30.621636+0000 mon.a (mon.0) 434 : cluster [WRN] Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:31.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[84777]: cluster 2024-04-13T07:43:30.597069+0000 mon.a (mon.0) 429 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:31.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[84777]: cluster 2024-04-13T07:43:30.597128+0000 mon.a (mon.0) 430 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:43:31.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[84777]: cluster 2024-04-13T07:43:30.608110+0000 mon.a (mon.0) 431 : cluster [INF] osd.1 [v2:172.21.15.89:6810/2607331631,v1:172.21.15.89:6811/2607331631] boot 2024-04-13T07:43:31.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[84777]: cluster 2024-04-13T07:43:30.608296+0000 mon.a (mon.0) 432 : cluster [DBG] osdmap e101: 8 total, 8 up, 8 in 2024-04-13T07:43:31.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[84777]: audit 2024-04-13T07:43:30.609128+0000 mon.a (mon.0) 433 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2024-04-13T07:43:31.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:31 smithi089 bash[84777]: cluster 2024-04-13T07:43:30.621636+0000 mon.a (mon.0) 434 : cluster [WRN] Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 5 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:32.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:32 smithi203 bash[68060]: cluster 2024-04-13T07:43:31.096345+0000 mgr.y (mgr.34107) 241 : cluster [DBG] pgmap v179: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:32.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:32 smithi203 bash[68060]: audit 2024-04-13T07:43:31.483262+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.89:0/57544096' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:32.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:32 smithi203 bash[68060]: cluster 2024-04-13T07:43:31.611632+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-04-13T07:43:32.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:43:32 smithi203 bash[60252]: ts=2024-04-13T07:43:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:43:32.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:32 smithi089 bash[82331]: cluster 2024-04-13T07:43:31.096345+0000 mgr.y (mgr.34107) 241 : cluster [DBG] pgmap v179: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:32.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:32 smithi089 bash[82331]: audit 2024-04-13T07:43:31.483262+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.89:0/57544096' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:32.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:32 smithi089 bash[82331]: cluster 2024-04-13T07:43:31.611632+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-04-13T07:43:32.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:32 smithi089 bash[84777]: cluster 2024-04-13T07:43:31.096345+0000 mgr.y (mgr.34107) 241 : cluster [DBG] pgmap v179: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 250 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2024-04-13T07:43:32.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:32 smithi089 bash[84777]: audit 2024-04-13T07:43:31.483262+0000 mon.c (mon.2) 7 : audit [DBG] from='client.? 172.21.15.89:0/57544096' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:32.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:32 smithi089 bash[84777]: cluster 2024-04-13T07:43:31.611632+0000 mon.a (mon.0) 435 : cluster [DBG] osdmap e102: 8 total, 8 up, 8 in 2024-04-13T07:43:34.251 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:34 smithi089 bash[82331]: cluster 2024-04-13T07:43:33.097355+0000 mgr.y (mgr.34107) 242 : cluster [DBG] pgmap v181: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:34.251 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:34 smithi089 bash[82331]: audit 2024-04-13T07:43:33.842934+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.89:0/2810175253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:34.251 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:34 smithi089 bash[84777]: cluster 2024-04-13T07:43:33.097355+0000 mgr.y (mgr.34107) 242 : cluster [DBG] pgmap v181: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:34.252 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:34 smithi089 bash[84777]: audit 2024-04-13T07:43:33.842934+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.89:0/2810175253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:34.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:34 smithi203 bash[68060]: cluster 2024-04-13T07:43:33.097355+0000 mgr.y (mgr.34107) 242 : cluster [DBG] pgmap v181: 73 pgs: 2 active+undersized+degraded, 10 active+undersized, 61 active+clean; 4.6 MiB data, 254 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:34.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:34 smithi203 bash[68060]: audit 2024-04-13T07:43:33.842934+0000 mon.c (mon.2) 8 : audit [DBG] from='client.? 172.21.15.89:0/2810175253' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:36.118 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:43:35] "GET /metrics HTTP/1.1" 200 34015 "" "Prometheus/2.43.0" 2024-04-13T07:43:36.501 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:36 smithi089 bash[82331]: cluster 2024-04-13T07:43:35.098212+0000 mgr.y (mgr.34107) 243 : cluster [DBG] pgmap v182: 73 pgs: 1 active+undersized+degraded, 2 active+undersized, 70 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-13T07:43:36.501 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:36 smithi089 bash[82331]: cluster 2024-04-13T07:43:35.623424+0000 mon.a (mon.0) 436 : cluster [WRN] Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:43:36.502 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:36 smithi089 bash[84777]: cluster 2024-04-13T07:43:35.098212+0000 mgr.y (mgr.34107) 243 : cluster [DBG] pgmap v182: 73 pgs: 1 active+undersized+degraded, 2 active+undersized, 70 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-13T07:43:36.502 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:36 smithi089 bash[84777]: cluster 2024-04-13T07:43:35.623424+0000 mon.a (mon.0) 436 : cluster [WRN] Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:43:36.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:36 smithi203 bash[68060]: cluster 2024-04-13T07:43:35.098212+0000 mgr.y (mgr.34107) 243 : cluster [DBG] pgmap v182: 73 pgs: 1 active+undersized+degraded, 2 active+undersized, 70 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-13T07:43:36.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:36 smithi203 bash[68060]: cluster 2024-04-13T07:43:35.623424+0000 mon.a (mon.0) 436 : cluster [WRN] Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:43:37.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:37 smithi203 bash[68060]: audit 2024-04-13T07:43:36.197151+0000 mon.a (mon.0) 437 : audit [DBG] from='client.? 172.21.15.89:0/2759266254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:37.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:37 smithi089 bash[84777]: audit 2024-04-13T07:43:36.197151+0000 mon.a (mon.0) 437 : audit [DBG] from='client.? 172.21.15.89:0/2759266254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:37.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:37 smithi089 bash[82331]: audit 2024-04-13T07:43:36.197151+0000 mon.a (mon.0) 437 : audit [DBG] from='client.? 172.21.15.89:0/2759266254' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: cluster 2024-04-13T07:43:37.098762+0000 mgr.y (mgr.34107) 244 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: cluster 2024-04-13T07:43:37.126036+0000 mon.a (mon.0) 438 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.156568+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.169800+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.172176+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.173665+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.183121+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.279996+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.285467+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.289136+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.291738+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.295782+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-13T07:43:38.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: audit 2024-04-13T07:43:37.296171+0000 mgr.y (mgr.34107) 245 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-13T07:43:38.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:38 smithi203 bash[68060]: cephadm 2024-04-13T07:43:37.298392+0000 mgr.y (mgr.34107) 246 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: cluster 2024-04-13T07:43:37.098762+0000 mgr.y (mgr.34107) 244 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: cluster 2024-04-13T07:43:37.126036+0000 mon.a (mon.0) 438 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.156568+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.169800+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.172176+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.173665+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.183121+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.279996+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.285467+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.289136+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.291738+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.532 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.295782+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: audit 2024-04-13T07:43:37.296171+0000 mgr.y (mgr.34107) 245 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[82331]: cephadm 2024-04-13T07:43:37.298392+0000 mgr.y (mgr.34107) 246 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: cluster 2024-04-13T07:43:37.098762+0000 mgr.y (mgr.34107) 244 : cluster [DBG] pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: cluster 2024-04-13T07:43:37.126036+0000 mon.a (mon.0) 438 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.156568+0000 mon.a (mon.0) 439 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.169800+0000 mon.a (mon.0) 440 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.172176+0000 mon.a (mon.0) 441 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.173665+0000 mon.a (mon.0) 442 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.183121+0000 mon.a (mon.0) 443 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.279996+0000 mon.a (mon.0) 444 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.285467+0000 mon.a (mon.0) 445 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.533 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.289136+0000 mon.a (mon.0) 446 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.291738+0000 mon.a (mon.0) 447 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:43:38.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.295782+0000 mon.a (mon.0) 448 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-13T07:43:38.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: audit 2024-04-13T07:43:37.296171+0000 mgr.y (mgr.34107) 245 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2024-04-13T07:43:38.534 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:38 smithi089 bash[84777]: cephadm 2024-04-13T07:43:37.298392+0000 mgr.y (mgr.34107) 246 : cephadm [INF] Upgrade: osd.2 is safe to restart 2024-04-13T07:43:39.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:39 smithi203 bash[68060]: audit 2024-04-13T07:43:38.561593+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.89:0/340601979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:39.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:39 smithi089 bash[82331]: audit 2024-04-13T07:43:38.561593+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.89:0/340601979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:39.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:39 smithi089 bash[84777]: audit 2024-04-13T07:43:38.561593+0000 mon.c (mon.2) 9 : audit [DBG] from='client.? 172.21.15.89:0/340601979' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:40.462 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[84777]: cluster 2024-04-13T07:43:39.099781+0000 mgr.y (mgr.34107) 247 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[84777]: cephadm 2024-04-13T07:43:39.167182+0000 mgr.y (mgr.34107) 248 : cephadm [INF] Upgrade: Updating osd.2 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[84777]: audit 2024-04-13T07:43:39.172458+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[84777]: audit 2024-04-13T07:43:39.175189+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[84777]: audit 2024-04-13T07:43:39.176365+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[84777]: cephadm 2024-04-13T07:43:39.180880+0000 mgr.y (mgr.34107) 249 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[82331]: cluster 2024-04-13T07:43:39.099781+0000 mgr.y (mgr.34107) 247 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[82331]: cephadm 2024-04-13T07:43:39.167182+0000 mgr.y (mgr.34107) 248 : cephadm [INF] Upgrade: Updating osd.2 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[82331]: audit 2024-04-13T07:43:39.172458+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[82331]: audit 2024-04-13T07:43:39.175189+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:43:40.463 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[82331]: audit 2024-04-13T07:43:39.176365+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:40.464 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:40 smithi089 bash[82331]: cephadm 2024-04-13T07:43:39.180880+0000 mgr.y (mgr.34107) 249 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-04-13T07:43:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:40 smithi203 bash[68060]: cluster 2024-04-13T07:43:39.099781+0000 mgr.y (mgr.34107) 247 : cluster [DBG] pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:40 smithi203 bash[68060]: cephadm 2024-04-13T07:43:39.167182+0000 mgr.y (mgr.34107) 248 : cephadm [INF] Upgrade: Updating osd.2 2024-04-13T07:43:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:40 smithi203 bash[68060]: audit 2024-04-13T07:43:39.172458+0000 mon.a (mon.0) 449 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:40 smithi203 bash[68060]: audit 2024-04-13T07:43:39.175189+0000 mon.a (mon.0) 450 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2024-04-13T07:43:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:40 smithi203 bash[68060]: audit 2024-04-13T07:43:39.176365+0000 mon.a (mon.0) 451 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:43:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:40 smithi203 bash[68060]: cephadm 2024-04-13T07:43:39.180880+0000 mgr.y (mgr.34107) 249 : cephadm [INF] Deploying daemon osd.2 on smithi089 2024-04-13T07:43:41.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:41 smithi203 bash[68060]: audit 2024-04-13T07:43:40.924710+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.89:0/3517443927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:41.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:41 smithi089 bash[82331]: audit 2024-04-13T07:43:40.924710+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.89:0/3517443927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:41.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:41 smithi089 bash[84777]: audit 2024-04-13T07:43:40.924710+0000 mon.c (mon.2) 10 : audit [DBG] from='client.? 172.21.15.89:0/3517443927' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:42.514 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:43:42 smithi203 bash[60252]: ts=2024-04-13T07:43:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:43:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:42 smithi203 bash[68060]: cluster 2024-04-13T07:43:41.100389+0000 mgr.y (mgr.34107) 250 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:42 smithi203 bash[68060]: audit 2024-04-13T07:43:42.077367+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:42 smithi203 bash[68060]: audit 2024-04-13T07:43:42.078801+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:42 smithi203 bash[68060]: audit 2024-04-13T07:43:42.172693+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:42.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:42 smithi089 bash[82331]: cluster 2024-04-13T07:43:41.100389+0000 mgr.y (mgr.34107) 250 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:42.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:42 smithi089 bash[82331]: audit 2024-04-13T07:43:42.077367+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:42.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:42 smithi089 bash[82331]: audit 2024-04-13T07:43:42.078801+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:42.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:42 smithi089 bash[82331]: audit 2024-04-13T07:43:42.172693+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:42.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:42 smithi089 bash[84777]: cluster 2024-04-13T07:43:41.100389+0000 mgr.y (mgr.34107) 250 : cluster [DBG] pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:42.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:42 smithi089 bash[84777]: audit 2024-04-13T07:43:42.077367+0000 mon.a (mon.0) 452 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:42.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:42 smithi089 bash[84777]: audit 2024-04-13T07:43:42.078801+0000 mon.a (mon.0) 453 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:42.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:42 smithi089 bash[84777]: audit 2024-04-13T07:43:42.172693+0000 mon.a (mon.0) 454 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:43.224 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.225 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.225 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.225 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.226 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.226 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.226 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.227 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.227 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:43.548 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:43 smithi089 systemd[1]: Stopping Ceph osd.2 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:43:43.548 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:43 smithi089 bash[45658]: debug 2024-04-13T07:43:43.349+0000 7fed39b32700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:43:43.548 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:43 smithi089 bash[45658]: debug 2024-04-13T07:43:43.349+0000 7fed39b32700 -1 osd.2 102 *** Got signal Terminated *** 2024-04-13T07:43:43.548 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:43 smithi089 bash[45658]: debug 2024-04-13T07:43:43.349+0000 7fed39b32700 -1 osd.2 102 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:43:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:44 smithi203 bash[68060]: cluster 2024-04-13T07:43:43.100921+0000 mgr.y (mgr.34107) 251 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:44 smithi203 bash[68060]: audit 2024-04-13T07:43:43.294938+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/1472197837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:44 smithi203 bash[68060]: cluster 2024-04-13T07:43:43.353313+0000 mon.a (mon.0) 455 : cluster [INF] osd.2 marked itself down and dead 2024-04-13T07:43:44.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:44 smithi089 bash[82331]: debug 2024-04-13T07:43:44.185+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:44.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:44 smithi089 bash[82331]: cluster 2024-04-13T07:43:43.100921+0000 mgr.y (mgr.34107) 251 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:44.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:44 smithi089 bash[82331]: audit 2024-04-13T07:43:43.294938+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/1472197837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:44.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:44 smithi089 bash[82331]: cluster 2024-04-13T07:43:43.353313+0000 mon.a (mon.0) 455 : cluster [INF] osd.2 marked itself down and dead 2024-04-13T07:43:44.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:44 smithi089 bash[84777]: cluster 2024-04-13T07:43:43.100921+0000 mgr.y (mgr.34107) 251 : cluster [DBG] pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:44.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:44 smithi089 bash[84777]: audit 2024-04-13T07:43:43.294938+0000 mon.c (mon.2) 11 : audit [DBG] from='client.? 172.21.15.89:0/1472197837' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:44.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:44 smithi089 bash[84777]: cluster 2024-04-13T07:43:43.353313+0000 mon.a (mon.0) 455 : cluster [INF] osd.2 marked itself down and dead 2024-04-13T07:43:45.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:45 smithi203 bash[68060]: cluster 2024-04-13T07:43:44.187079+0000 mon.a (mon.0) 456 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:43:45.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:45 smithi203 bash[68060]: cluster 2024-04-13T07:43:44.187156+0000 mon.a (mon.0) 457 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:45.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:45 smithi203 bash[68060]: cluster 2024-04-13T07:43:44.208342+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e103: 8 total, 7 up, 8 in 2024-04-13T07:43:45.547 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[100540]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-2 2024-04-13T07:43:45.547 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[100613]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-2 2024-04-13T07:43:45.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[82331]: cluster 2024-04-13T07:43:44.187079+0000 mon.a (mon.0) 456 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:43:45.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[82331]: cluster 2024-04-13T07:43:44.187156+0000 mon.a (mon.0) 457 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:45.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[82331]: cluster 2024-04-13T07:43:44.208342+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e103: 8 total, 7 up, 8 in 2024-04-13T07:43:45.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[84777]: cluster 2024-04-13T07:43:44.187079+0000 mon.a (mon.0) 456 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:43:45.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[84777]: cluster 2024-04-13T07:43:44.187156+0000 mon.a (mon.0) 457 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:45.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[84777]: cluster 2024-04-13T07:43:44.208342+0000 mon.a (mon.0) 458 : cluster [DBG] osdmap e103: 8 total, 7 up, 8 in 2024-04-13T07:43:46.200 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:43:45] "GET /metrics HTTP/1.1" 200 34008 "" "Prometheus/2.43.0" 2024-04-13T07:43:46.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:46 smithi203 bash[68060]: cluster 2024-04-13T07:43:45.101596+0000 mgr.y (mgr.34107) 252 : cluster [DBG] pgmap v188: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:46 smithi203 bash[68060]: cluster 2024-04-13T07:43:45.207930+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e104: 8 total, 7 up, 8 in 2024-04-13T07:43:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:46 smithi203 bash[68060]: audit 2024-04-13T07:43:45.648727+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.89:0/2510078356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:46.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:46 smithi089 bash[82331]: cluster 2024-04-13T07:43:45.101596+0000 mgr.y (mgr.34107) 252 : cluster [DBG] pgmap v188: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:46.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:46 smithi089 bash[82331]: cluster 2024-04-13T07:43:45.207930+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e104: 8 total, 7 up, 8 in 2024-04-13T07:43:46.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:46 smithi089 bash[82331]: audit 2024-04-13T07:43:45.648727+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.89:0/2510078356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:46.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:46 smithi089 bash[84777]: cluster 2024-04-13T07:43:45.101596+0000 mgr.y (mgr.34107) 252 : cluster [DBG] pgmap v188: 73 pgs: 3 stale+active+clean, 70 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:43:46.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:46 smithi089 bash[84777]: cluster 2024-04-13T07:43:45.207930+0000 mon.a (mon.0) 459 : cluster [DBG] osdmap e104: 8 total, 7 up, 8 in 2024-04-13T07:43:46.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:46 smithi089 bash[84777]: audit 2024-04-13T07:43:45.648727+0000 mon.c (mon.2) 12 : audit [DBG] from='client.? 172.21.15.89:0/2510078356' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:47.486 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:47 smithi089 bash[84777]: cluster 2024-04-13T07:43:47.206884+0000 mon.a (mon.0) 460 : cluster [WRN] Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:43:47.487 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.487 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.487 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.2.service: Deactivated successfully. 2024-04-13T07:43:47.487 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: Stopped Ceph osd.2 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:43:47.487 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.488 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.488 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.488 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:47 smithi089 bash[82331]: cluster 2024-04-13T07:43:47.206884+0000 mon.a (mon.0) 460 : cluster [WRN] Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:43:47.488 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.489 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.489 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.489 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:43:47.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:47 smithi203 bash[68060]: cluster 2024-04-13T07:43:47.206884+0000 mon.a (mon.0) 460 : cluster [WRN] Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:43:47.797 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:47 smithi089 systemd[1]: Started Ceph osd.2 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[82331]: cluster 2024-04-13T07:43:47.102499+0000 mgr.y (mgr.34107) 253 : cluster [DBG] pgmap v190: 73 pgs: 1 active+undersized+degraded, 3 stale+active+clean, 1 active+undersized, 68 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[82331]: audit 2024-04-13T07:43:47.541618+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[82331]: audit 2024-04-13T07:43:47.548622+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[82331]: audit 2024-04-13T07:43:47.566968+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[82331]: audit 2024-04-13T07:43:47.999256+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.89:0/3826400325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[84777]: cluster 2024-04-13T07:43:47.102499+0000 mgr.y (mgr.34107) 253 : cluster [DBG] pgmap v190: 73 pgs: 1 active+undersized+degraded, 3 stale+active+clean, 1 active+undersized, 68 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[84777]: audit 2024-04-13T07:43:47.541618+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[84777]: audit 2024-04-13T07:43:47.548622+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:48.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[84777]: audit 2024-04-13T07:43:47.566968+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[84777]: audit 2024-04-13T07:43:47.999256+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.89:0/3826400325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:49.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:48 smithi203 bash[68060]: cluster 2024-04-13T07:43:47.102499+0000 mgr.y (mgr.34107) 253 : cluster [DBG] pgmap v190: 73 pgs: 1 active+undersized+degraded, 3 stale+active+clean, 1 active+undersized, 68 active+clean; 4.6 MiB data, 258 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:43:49.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:48 smithi203 bash[68060]: audit 2024-04-13T07:43:47.541618+0000 mon.a (mon.0) 461 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:49.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:48 smithi203 bash[68060]: audit 2024-04-13T07:43:47.548622+0000 mon.a (mon.0) 462 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:49.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:48 smithi203 bash[68060]: audit 2024-04-13T07:43:47.566968+0000 mon.a (mon.0) 463 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:43:49.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:48 smithi203 bash[68060]: audit 2024-04-13T07:43:47.999256+0000 mon.c (mon.2) 13 : audit [DBG] from='client.? 172.21.15.89:0/3826400325' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:49.157 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[100786]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-13T07:43:49.157 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[100786]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-2 --no-mon-config --dev /dev/mapper/vg_nvme-lv_2 2024-04-13T07:43:49.157 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[100786]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_2 2024-04-13T07:43:49.157 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[100786]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2024-04-13T07:43:49.157 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[100786]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_2 /var/lib/ceph/osd/ceph-2/block 2024-04-13T07:43:49.157 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[100786]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2024-04-13T07:43:49.157 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:48 smithi089 bash[100786]: --> ceph-volume raw activate successful for osd ID: 2 2024-04-13T07:43:50.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:50 smithi089 bash[82331]: cluster 2024-04-13T07:43:49.103580+0000 mgr.y (mgr.34107) 254 : cluster [DBG] pgmap v191: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 11 active+undersized, 58 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:50.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:50 smithi089 bash[82331]: audit 2024-04-13T07:43:50.358467+0000 mon.a (mon.0) 464 : audit [DBG] from='client.? 172.21.15.89:0/3313662761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:50.798 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:50 smithi089 bash[101030]: debug 2024-04-13T07:43:50.441+0000 7fea69c9a700 1 -- 172.21.15.89:0/2958175325 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x560d10597680 con 0x560d105d4c00 2024-04-13T07:43:50.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:50 smithi089 bash[84777]: cluster 2024-04-13T07:43:49.103580+0000 mgr.y (mgr.34107) 254 : cluster [DBG] pgmap v191: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 11 active+undersized, 58 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:50.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:50 smithi089 bash[84777]: audit 2024-04-13T07:43:50.358467+0000 mon.a (mon.0) 464 : audit [DBG] from='client.? 172.21.15.89:0/3313662761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:51.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:50 smithi203 bash[68060]: cluster 2024-04-13T07:43:49.103580+0000 mgr.y (mgr.34107) 254 : cluster [DBG] pgmap v191: 73 pgs: 3 active+undersized+degraded, 1 stale+active+clean, 11 active+undersized, 58 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:51.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:50 smithi203 bash[68060]: audit 2024-04-13T07:43:50.358467+0000 mon.a (mon.0) 464 : audit [DBG] from='client.? 172.21.15.89:0/3313662761' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:51.297 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:50 smithi089 bash[101030]: debug 2024-04-13T07:43:50.989+0000 7fea72e327c0 -1 Falling back to public interface 2024-04-13T07:43:52.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:52 smithi203 bash[68060]: cluster 2024-04-13T07:43:51.104583+0000 mgr.y (mgr.34107) 255 : cluster [DBG] pgmap v192: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:52.765 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:43:52 smithi203 bash[60252]: ts=2024-04-13T07:43:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:43:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:52 smithi089 bash[82331]: cluster 2024-04-13T07:43:51.104583+0000 mgr.y (mgr.34107) 255 : cluster [DBG] pgmap v192: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:52.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:52 smithi089 bash[84777]: cluster 2024-04-13T07:43:51.104583+0000 mgr.y (mgr.34107) 255 : cluster [DBG] pgmap v192: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:54.010 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:53 smithi089 bash[84777]: audit 2024-04-13T07:43:52.717990+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/2379127457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:54.010 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:53 smithi089 bash[82331]: audit 2024-04-13T07:43:52.717990+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/2379127457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:54.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:53 smithi203 bash[68060]: audit 2024-04-13T07:43:52.717990+0000 mon.c (mon.2) 14 : audit [DBG] from='client.? 172.21.15.89:0/2379127457' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:54.298 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:54 smithi089 bash[101030]: debug 2024-04-13T07:43:54.013+0000 7fea72e327c0 -1 osd.2 102 log_to_monitors true 2024-04-13T07:43:55.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:54 smithi203 bash[68060]: cluster 2024-04-13T07:43:53.105355+0000 mgr.y (mgr.34107) 256 : cluster [DBG] pgmap v193: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:55.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:54 smithi203 bash[68060]: audit 2024-04-13T07:43:54.021360+0000 mon.a (mon.0) 465 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:43:55.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:54 smithi089 bash[84777]: cluster 2024-04-13T07:43:53.105355+0000 mgr.y (mgr.34107) 256 : cluster [DBG] pgmap v193: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:55.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:54 smithi089 bash[84777]: audit 2024-04-13T07:43:54.021360+0000 mon.a (mon.0) 465 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:43:55.048 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:43:54 smithi089 bash[101030]: debug 2024-04-13T07:43:54.597+0000 7fea6428a700 -1 osd.2 102 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-13T07:43:55.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:54 smithi089 bash[82331]: cluster 2024-04-13T07:43:53.105355+0000 mgr.y (mgr.34107) 256 : cluster [DBG] pgmap v193: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 259 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:55.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:54 smithi089 bash[82331]: audit 2024-04-13T07:43:54.021360+0000 mon.a (mon.0) 465 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2024-04-13T07:43:55.850 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[84777]: audit 2024-04-13T07:43:54.564416+0000 mon.a (mon.0) 466 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-13T07:43:55.850 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[84777]: cluster 2024-04-13T07:43:54.567555+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e105: 8 total, 7 up, 8 in 2024-04-13T07:43:55.850 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[84777]: audit 2024-04-13T07:43:54.568518+0000 mon.a (mon.0) 468 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:43:55.850 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[84777]: audit 2024-04-13T07:43:54.903838+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:55.850 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[84777]: audit 2024-04-13T07:43:54.913156+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:55.851 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[84777]: audit 2024-04-13T07:43:55.062243+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/375020402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:55.851 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[82331]: debug 2024-04-13T07:43:55.569+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:55.851 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[82331]: audit 2024-04-13T07:43:54.564416+0000 mon.a (mon.0) 466 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-13T07:43:55.851 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[82331]: cluster 2024-04-13T07:43:54.567555+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e105: 8 total, 7 up, 8 in 2024-04-13T07:43:55.851 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[82331]: audit 2024-04-13T07:43:54.568518+0000 mon.a (mon.0) 468 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:43:55.851 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[82331]: audit 2024-04-13T07:43:54.903838+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:55.851 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[82331]: audit 2024-04-13T07:43:54.913156+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:55.851 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[82331]: audit 2024-04-13T07:43:55.062243+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/375020402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:56.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:55 smithi203 bash[68060]: audit 2024-04-13T07:43:54.564416+0000 mon.a (mon.0) 466 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2024-04-13T07:43:56.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:55 smithi203 bash[68060]: cluster 2024-04-13T07:43:54.567555+0000 mon.a (mon.0) 467 : cluster [DBG] osdmap e105: 8 total, 7 up, 8 in 2024-04-13T07:43:56.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:55 smithi203 bash[68060]: audit 2024-04-13T07:43:54.568518+0000 mon.a (mon.0) 468 : audit [INF] from='osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:43:56.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:55 smithi203 bash[68060]: audit 2024-04-13T07:43:54.903838+0000 mon.a (mon.0) 469 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:56.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:55 smithi203 bash[68060]: audit 2024-04-13T07:43:54.913156+0000 mon.a (mon.0) 470 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:56.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:55 smithi203 bash[68060]: audit 2024-04-13T07:43:55.062243+0000 mon.c (mon.2) 15 : audit [DBG] from='client.? 172.21.15.89:0/375020402' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:56.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:43:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:43:55] "GET /metrics HTTP/1.1" 200 34008 "" "Prometheus/2.43.0" 2024-04-13T07:43:57.014 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: cluster 2024-04-13T07:43:55.105832+0000 mgr.y (mgr.34107) 257 : cluster [DBG] pgmap v195: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: cluster 2024-04-13T07:43:55.573392+0000 mon.a (mon.0) 471 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: cluster 2024-04-13T07:43:55.573452+0000 mon.a (mon.0) 472 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: cluster 2024-04-13T07:43:55.581456+0000 mon.a (mon.0) 473 : cluster [INF] osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818] boot 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: cluster 2024-04-13T07:43:55.581541+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: audit 2024-04-13T07:43:55.582079+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: cluster 2024-04-13T07:43:55.627601+0000 mon.a (mon.0) 476 : cluster [WRN] Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: audit 2024-04-13T07:43:56.011432+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: audit 2024-04-13T07:43:56.020405+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:57.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:56 smithi203 bash[68060]: cluster 2024-04-13T07:43:56.583021+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-04-13T07:43:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: cluster 2024-04-13T07:43:55.105832+0000 mgr.y (mgr.34107) 257 : cluster [DBG] pgmap v195: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: cluster 2024-04-13T07:43:55.573392+0000 mon.a (mon.0) 471 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:57.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: cluster 2024-04-13T07:43:55.573452+0000 mon.a (mon.0) 472 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: cluster 2024-04-13T07:43:55.581456+0000 mon.a (mon.0) 473 : cluster [INF] osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818] boot 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: cluster 2024-04-13T07:43:55.581541+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: audit 2024-04-13T07:43:55.582079+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: cluster 2024-04-13T07:43:55.627601+0000 mon.a (mon.0) 476 : cluster [WRN] Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: audit 2024-04-13T07:43:56.011432+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: audit 2024-04-13T07:43:56.020405+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[82331]: cluster 2024-04-13T07:43:56.583021+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: cluster 2024-04-13T07:43:55.105832+0000 mgr.y (mgr.34107) 257 : cluster [DBG] pgmap v195: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: cluster 2024-04-13T07:43:55.573392+0000 mon.a (mon.0) 471 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:43:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: cluster 2024-04-13T07:43:55.573452+0000 mon.a (mon.0) 472 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:43:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: cluster 2024-04-13T07:43:55.581456+0000 mon.a (mon.0) 473 : cluster [INF] osd.2 [v2:172.21.15.89:6818/2720399818,v1:172.21.15.89:6819/2720399818] boot 2024-04-13T07:43:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: cluster 2024-04-13T07:43:55.581541+0000 mon.a (mon.0) 474 : cluster [DBG] osdmap e106: 8 total, 8 up, 8 in 2024-04-13T07:43:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: audit 2024-04-13T07:43:55.582079+0000 mon.a (mon.0) 475 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2024-04-13T07:43:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: cluster 2024-04-13T07:43:55.627601+0000 mon.a (mon.0) 476 : cluster [WRN] Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 3 pgs degraded (PG_DEGRADED) 2024-04-13T07:43:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: audit 2024-04-13T07:43:56.011432+0000 mon.a (mon.0) 477 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: audit 2024-04-13T07:43:56.020405+0000 mon.a (mon.0) 478 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:57.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:56 smithi089 bash[84777]: cluster 2024-04-13T07:43:56.583021+0000 mon.a (mon.0) 479 : cluster [DBG] osdmap e107: 8 total, 8 up, 8 in 2024-04-13T07:43:58.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:58 smithi203 bash[68060]: audit 2024-04-13T07:43:57.079031+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:58 smithi203 bash[68060]: audit 2024-04-13T07:43:57.081348+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:58 smithi203 bash[68060]: cluster 2024-04-13T07:43:57.109709+0000 mgr.y (mgr.34107) 258 : cluster [DBG] pgmap v198: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:58 smithi203 bash[68060]: audit 2024-04-13T07:43:57.187439+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:43:58 smithi203 bash[68060]: audit 2024-04-13T07:43:57.432339+0000 mon.a (mon.0) 483 : audit [DBG] from='client.? 172.21.15.89:0/1117813146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[82331]: audit 2024-04-13T07:43:57.079031+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[82331]: audit 2024-04-13T07:43:57.081348+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[82331]: cluster 2024-04-13T07:43:57.109709+0000 mgr.y (mgr.34107) 258 : cluster [DBG] pgmap v198: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[82331]: audit 2024-04-13T07:43:57.187439+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[82331]: audit 2024-04-13T07:43:57.432339+0000 mon.a (mon.0) 483 : audit [DBG] from='client.? 172.21.15.89:0/1117813146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:43:58.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[84777]: audit 2024-04-13T07:43:57.079031+0000 mon.a (mon.0) 480 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:58.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[84777]: audit 2024-04-13T07:43:57.081348+0000 mon.a (mon.0) 481 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:43:58.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[84777]: cluster 2024-04-13T07:43:57.109709+0000 mgr.y (mgr.34107) 258 : cluster [DBG] pgmap v198: 73 pgs: 3 active+undersized+degraded, 16 active+undersized, 54 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:43:58.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[84777]: audit 2024-04-13T07:43:57.187439+0000 mon.a (mon.0) 482 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:43:58.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:43:58 smithi089 bash[84777]: audit 2024-04-13T07:43:57.432339+0000 mon.a (mon.0) 483 : audit [DBG] from='client.? 172.21.15.89:0/1117813146' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:00.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:00 smithi203 bash[68060]: cluster 2024-04-13T07:43:59.110748+0000 mgr.y (mgr.34107) 259 : cluster [DBG] pgmap v199: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:44:00.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:00 smithi203 bash[68060]: audit 2024-04-13T07:43:59.796660+0000 mon.a (mon.0) 484 : audit [DBG] from='client.? 172.21.15.89:0/630299546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:00.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:00 smithi089 bash[82331]: cluster 2024-04-13T07:43:59.110748+0000 mgr.y (mgr.34107) 259 : cluster [DBG] pgmap v199: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:44:00.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:00 smithi089 bash[82331]: audit 2024-04-13T07:43:59.796660+0000 mon.a (mon.0) 484 : audit [DBG] from='client.? 172.21.15.89:0/630299546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:00.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:00 smithi089 bash[84777]: cluster 2024-04-13T07:43:59.110748+0000 mgr.y (mgr.34107) 259 : cluster [DBG] pgmap v199: 73 pgs: 1 active+undersized+degraded, 8 active+undersized, 64 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2024-04-13T07:44:00.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:00 smithi089 bash[84777]: audit 2024-04-13T07:43:59.796660+0000 mon.a (mon.0) 484 : audit [DBG] from='client.? 172.21.15.89:0/630299546' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:01.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:01 smithi089 bash[84777]: cluster 2024-04-13T07:44:00.628809+0000 mon.a (mon.0) 485 : cluster [WRN] Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:44:01.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:01 smithi089 bash[82331]: cluster 2024-04-13T07:44:00.628809+0000 mon.a (mon.0) 485 : cluster [WRN] Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:44:01.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:01 smithi203 bash[68060]: cluster 2024-04-13T07:44:00.628809+0000 mon.a (mon.0) 485 : cluster [WRN] Health check update: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:44:02.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:02 smithi089 bash[84777]: cluster 2024-04-13T07:44:01.111580+0000 mgr.y (mgr.34107) 260 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:02.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:02 smithi089 bash[84777]: cluster 2024-04-13T07:44:01.166355+0000 mon.a (mon.0) 486 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-04-13T07:44:02.230 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:02 smithi089 bash[84777]: audit 2024-04-13T07:44:02.158046+0000 mon.a (mon.0) 487 : audit [DBG] from='client.? 172.21.15.89:0/83715080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:02.230 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:02 smithi089 bash[82331]: cluster 2024-04-13T07:44:01.111580+0000 mgr.y (mgr.34107) 260 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:02.230 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:02 smithi089 bash[82331]: cluster 2024-04-13T07:44:01.166355+0000 mon.a (mon.0) 486 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-04-13T07:44:02.231 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:02 smithi089 bash[82331]: audit 2024-04-13T07:44:02.158046+0000 mon.a (mon.0) 487 : audit [DBG] from='client.? 172.21.15.89:0/83715080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:02.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:02 smithi203 bash[68060]: cluster 2024-04-13T07:44:01.111580+0000 mgr.y (mgr.34107) 260 : cluster [DBG] pgmap v200: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:02.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:02 smithi203 bash[68060]: cluster 2024-04-13T07:44:01.166355+0000 mon.a (mon.0) 486 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/84 objects degraded (2.381%), 1 pg degraded) 2024-04-13T07:44:02.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:02 smithi203 bash[68060]: audit 2024-04-13T07:44:02.158046+0000 mon.a (mon.0) 487 : audit [DBG] from='client.? 172.21.15.89:0/83715080' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:02.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:44:02 smithi203 bash[60252]: ts=2024-04-13T07:44:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:44:04.424 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:04 smithi089 bash[82331]: cluster 2024-04-13T07:44:03.112531+0000 mgr.y (mgr.34107) 261 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:04.424 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:04 smithi089 bash[84777]: cluster 2024-04-13T07:44:03.112531+0000 mgr.y (mgr.34107) 261 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:04.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:04 smithi203 bash[68060]: cluster 2024-04-13T07:44:03.112531+0000 mgr.y (mgr.34107) 261 : cluster [DBG] pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:05.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:05 smithi203 bash[68060]: audit 2024-04-13T07:44:04.504211+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/2177013516' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:05.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:05 smithi203 bash[68060]: audit 2024-04-13T07:44:05.122683+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:05.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:05 smithi203 bash[68060]: audit 2024-04-13T07:44:05.136227+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:05.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:05 smithi203 bash[68060]: audit 2024-04-13T07:44:05.138862+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:05.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:05 smithi203 bash[68060]: audit 2024-04-13T07:44:05.140588+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:44:05.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:05 smithi203 bash[68060]: audit 2024-04-13T07:44:05.152732+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[82331]: audit 2024-04-13T07:44:04.504211+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/2177013516' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[82331]: audit 2024-04-13T07:44:05.122683+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[82331]: audit 2024-04-13T07:44:05.136227+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[82331]: audit 2024-04-13T07:44:05.138862+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[82331]: audit 2024-04-13T07:44:05.140588+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[82331]: audit 2024-04-13T07:44:05.152732+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[84777]: audit 2024-04-13T07:44:04.504211+0000 mon.c (mon.2) 16 : audit [DBG] from='client.? 172.21.15.89:0/2177013516' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[84777]: audit 2024-04-13T07:44:05.122683+0000 mon.a (mon.0) 488 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[84777]: audit 2024-04-13T07:44:05.136227+0000 mon.a (mon.0) 489 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[84777]: audit 2024-04-13T07:44:05.138862+0000 mon.a (mon.0) 490 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:05.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[84777]: audit 2024-04-13T07:44:05.140588+0000 mon.a (mon.0) 491 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:44:05.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[84777]: audit 2024-04-13T07:44:05.152732+0000 mon.a (mon.0) 492 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:06.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:44:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:44:05] "GET /metrics HTTP/1.1" 200 33862 "" "Prometheus/2.43.0" 2024-04-13T07:44:06.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:06 smithi203 bash[68060]: cluster 2024-04-13T07:44:05.113544+0000 mgr.y (mgr.34107) 262 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:06 smithi203 bash[68060]: audit 2024-04-13T07:44:05.272692+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:06 smithi203 bash[68060]: audit 2024-04-13T07:44:05.276569+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:06 smithi203 bash[68060]: audit 2024-04-13T07:44:05.279041+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:06 smithi203 bash[68060]: audit 2024-04-13T07:44:05.280950+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:06 smithi203 bash[68060]: audit 2024-04-13T07:44:05.283351+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-13T07:44:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:06 smithi203 bash[68060]: audit 2024-04-13T07:44:05.283636+0000 mgr.y (mgr.34107) 263 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-13T07:44:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:06 smithi203 bash[68060]: cephadm 2024-04-13T07:44:05.285262+0000 mgr.y (mgr.34107) 264 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-04-13T07:44:06.537 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[84777]: cluster 2024-04-13T07:44:05.113544+0000 mgr.y (mgr.34107) 262 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:06.537 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[84777]: audit 2024-04-13T07:44:05.272692+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:06.537 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[84777]: audit 2024-04-13T07:44:05.276569+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.537 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[84777]: audit 2024-04-13T07:44:05.279041+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.537 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[84777]: audit 2024-04-13T07:44:05.280950+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.537 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[84777]: audit 2024-04-13T07:44:05.283351+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-13T07:44:06.537 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[84777]: audit 2024-04-13T07:44:05.283636+0000 mgr.y (mgr.34107) 263 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-13T07:44:06.537 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[84777]: cephadm 2024-04-13T07:44:05.285262+0000 mgr.y (mgr.34107) 264 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-04-13T07:44:06.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[82331]: cluster 2024-04-13T07:44:05.113544+0000 mgr.y (mgr.34107) 262 : cluster [DBG] pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:06.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[82331]: audit 2024-04-13T07:44:05.272692+0000 mon.a (mon.0) 493 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:06.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[82331]: audit 2024-04-13T07:44:05.276569+0000 mon.a (mon.0) 494 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[82331]: audit 2024-04-13T07:44:05.279041+0000 mon.a (mon.0) 495 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[82331]: audit 2024-04-13T07:44:05.280950+0000 mon.a (mon.0) 496 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:06.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[82331]: audit 2024-04-13T07:44:05.283351+0000 mon.a (mon.0) 497 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-13T07:44:06.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[82331]: audit 2024-04-13T07:44:05.283636+0000 mgr.y (mgr.34107) 263 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2024-04-13T07:44:06.538 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:06 smithi089 bash[82331]: cephadm 2024-04-13T07:44:05.285262+0000 mgr.y (mgr.34107) 264 : cephadm [INF] Upgrade: osd.3 is safe to restart 2024-04-13T07:44:07.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:07 smithi203 bash[68060]: audit 2024-04-13T07:44:06.872336+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/3143957974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:07.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:07 smithi089 bash[82331]: audit 2024-04-13T07:44:06.872336+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/3143957974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:07.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:07 smithi089 bash[84777]: audit 2024-04-13T07:44:06.872336+0000 mon.c (mon.2) 17 : audit [DBG] from='client.? 172.21.15.89:0/3143957974' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:08.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:08 smithi203 bash[68060]: cluster 2024-04-13T07:44:07.114271+0000 mgr.y (mgr.34107) 265 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:08 smithi203 bash[68060]: audit 2024-04-13T07:44:07.211784+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:08 smithi203 bash[68060]: cephadm 2024-04-13T07:44:07.354824+0000 mgr.y (mgr.34107) 266 : cephadm [INF] Upgrade: Updating osd.3 2024-04-13T07:44:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:08 smithi203 bash[68060]: audit 2024-04-13T07:44:07.361879+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:08 smithi203 bash[68060]: audit 2024-04-13T07:44:07.365856+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:44:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:08 smithi203 bash[68060]: audit 2024-04-13T07:44:07.367212+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:08 smithi203 bash[68060]: cephadm 2024-04-13T07:44:07.371962+0000 mgr.y (mgr.34107) 267 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-04-13T07:44:08.546 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[84777]: cluster 2024-04-13T07:44:07.114271+0000 mgr.y (mgr.34107) 265 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:08.546 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[84777]: audit 2024-04-13T07:44:07.211784+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:08.546 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[84777]: cephadm 2024-04-13T07:44:07.354824+0000 mgr.y (mgr.34107) 266 : cephadm [INF] Upgrade: Updating osd.3 2024-04-13T07:44:08.546 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[84777]: audit 2024-04-13T07:44:07.361879+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[84777]: audit 2024-04-13T07:44:07.365856+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[84777]: audit 2024-04-13T07:44:07.367212+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[84777]: cephadm 2024-04-13T07:44:07.371962+0000 mgr.y (mgr.34107) 267 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[82331]: cluster 2024-04-13T07:44:07.114271+0000 mgr.y (mgr.34107) 265 : cluster [DBG] pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[82331]: audit 2024-04-13T07:44:07.211784+0000 mon.a (mon.0) 498 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[82331]: cephadm 2024-04-13T07:44:07.354824+0000 mgr.y (mgr.34107) 266 : cephadm [INF] Upgrade: Updating osd.3 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[82331]: audit 2024-04-13T07:44:07.361879+0000 mon.a (mon.0) 499 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[82331]: audit 2024-04-13T07:44:07.365856+0000 mon.a (mon.0) 500 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2024-04-13T07:44:08.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[82331]: audit 2024-04-13T07:44:07.367212+0000 mon.a (mon.0) 501 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:08.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:08 smithi089 bash[82331]: cephadm 2024-04-13T07:44:07.371962+0000 mgr.y (mgr.34107) 267 : cephadm [INF] Deploying daemon osd.3 on smithi089 2024-04-13T07:44:10.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:10 smithi089 bash[82331]: cluster 2024-04-13T07:44:09.115202+0000 mgr.y (mgr.34107) 268 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:10.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:10 smithi089 bash[82331]: audit 2024-04-13T07:44:09.215650+0000 mon.a (mon.0) 502 : audit [DBG] from='client.? 172.21.15.89:0/3307103104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:10.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:10 smithi089 bash[84777]: cluster 2024-04-13T07:44:09.115202+0000 mgr.y (mgr.34107) 268 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:10.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:10 smithi089 bash[84777]: audit 2024-04-13T07:44:09.215650+0000 mon.a (mon.0) 502 : audit [DBG] from='client.? 172.21.15.89:0/3307103104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:10.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:10 smithi203 bash[68060]: cluster 2024-04-13T07:44:09.115202+0000 mgr.y (mgr.34107) 268 : cluster [DBG] pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:10.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:10 smithi203 bash[68060]: audit 2024-04-13T07:44:09.215650+0000 mon.a (mon.0) 502 : audit [DBG] from='client.? 172.21.15.89:0/3307103104' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:11.410 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.410 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.411 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.411 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.411 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.412 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.412 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.412 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.412 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:11.798 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:11 smithi089 systemd[1]: Stopping Ceph osd.3 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:44:11.798 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:11 smithi089 bash[51546]: debug 2024-04-13T07:44:11.540+0000 7facc949e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:44:11.798 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:11 smithi089 bash[51546]: debug 2024-04-13T07:44:11.540+0000 7facc949e700 -1 osd.3 107 *** Got signal Terminated *** 2024-04-13T07:44:11.798 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:11 smithi089 bash[51546]: debug 2024-04-13T07:44:11.540+0000 7facc949e700 -1 osd.3 107 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:44:12.514 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:44:12 smithi203 bash[60252]: ts=2024-04-13T07:44:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:44:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:12 smithi203 bash[68060]: cluster 2024-04-13T07:44:11.115812+0000 mgr.y (mgr.34107) 269 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:12 smithi203 bash[68060]: cluster 2024-04-13T07:44:11.545278+0000 mon.a (mon.0) 503 : cluster [INF] osd.3 marked itself down and dead 2024-04-13T07:44:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:12 smithi203 bash[68060]: audit 2024-04-13T07:44:11.585086+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/839429598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:12 smithi203 bash[68060]: audit 2024-04-13T07:44:12.081408+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:12.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:12 smithi203 bash[68060]: audit 2024-04-13T07:44:12.084046+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[82331]: debug 2024-04-13T07:44:12.212+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[82331]: cluster 2024-04-13T07:44:11.115812+0000 mgr.y (mgr.34107) 269 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[82331]: cluster 2024-04-13T07:44:11.545278+0000 mon.a (mon.0) 503 : cluster [INF] osd.3 marked itself down and dead 2024-04-13T07:44:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[82331]: audit 2024-04-13T07:44:11.585086+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/839429598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:12.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[82331]: audit 2024-04-13T07:44:12.081408+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:12.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[82331]: audit 2024-04-13T07:44:12.084046+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:12.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[84777]: cluster 2024-04-13T07:44:11.115812+0000 mgr.y (mgr.34107) 269 : cluster [DBG] pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:12.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[84777]: cluster 2024-04-13T07:44:11.545278+0000 mon.a (mon.0) 503 : cluster [INF] osd.3 marked itself down and dead 2024-04-13T07:44:12.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[84777]: audit 2024-04-13T07:44:11.585086+0000 mon.c (mon.2) 18 : audit [DBG] from='client.? 172.21.15.89:0/839429598' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:12.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[84777]: audit 2024-04-13T07:44:12.081408+0000 mon.a (mon.0) 504 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:12.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:12 smithi089 bash[84777]: audit 2024-04-13T07:44:12.084046+0000 mon.a (mon.0) 505 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:13.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:13 smithi203 bash[68060]: cluster 2024-04-13T07:44:12.216832+0000 mon.a (mon.0) 506 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:44:13.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:13 smithi203 bash[68060]: cluster 2024-04-13T07:44:12.216890+0000 mon.a (mon.0) 507 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:13.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:13 smithi203 bash[68060]: cluster 2024-04-13T07:44:12.234275+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e108: 8 total, 7 up, 8 in 2024-04-13T07:44:13.548 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:13 smithi089 bash[104824]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-3 2024-04-13T07:44:13.548 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:13 smithi089 bash[104890]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-3 2024-04-13T07:44:13.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:13 smithi089 bash[82331]: cluster 2024-04-13T07:44:12.216832+0000 mon.a (mon.0) 506 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:44:13.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:13 smithi089 bash[82331]: cluster 2024-04-13T07:44:12.216890+0000 mon.a (mon.0) 507 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:13.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:13 smithi089 bash[82331]: cluster 2024-04-13T07:44:12.234275+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e108: 8 total, 7 up, 8 in 2024-04-13T07:44:13.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:13 smithi089 bash[84777]: cluster 2024-04-13T07:44:12.216832+0000 mon.a (mon.0) 506 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:44:13.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:13 smithi089 bash[84777]: cluster 2024-04-13T07:44:12.216890+0000 mon.a (mon.0) 507 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:13.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:13 smithi089 bash[84777]: cluster 2024-04-13T07:44:12.234275+0000 mon.a (mon.0) 508 : cluster [DBG] osdmap e108: 8 total, 7 up, 8 in 2024-04-13T07:44:14.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:14 smithi203 bash[68060]: cluster 2024-04-13T07:44:13.116833+0000 mgr.y (mgr.34107) 270 : cluster [DBG] pgmap v207: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:14.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:14 smithi203 bash[68060]: cluster 2024-04-13T07:44:13.226339+0000 mon.a (mon.0) 509 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-13T07:44:14.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:14 smithi203 bash[68060]: cluster 2024-04-13T07:44:13.237635+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e109: 8 total, 7 up, 8 in 2024-04-13T07:44:14.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:14 smithi203 bash[68060]: audit 2024-04-13T07:44:13.942747+0000 mon.a (mon.0) 511 : audit [DBG] from='client.? 172.21.15.89:0/653019870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:14.526 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:14 smithi089 bash[82331]: cluster 2024-04-13T07:44:13.116833+0000 mgr.y (mgr.34107) 270 : cluster [DBG] pgmap v207: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:14.526 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:14 smithi089 bash[82331]: cluster 2024-04-13T07:44:13.226339+0000 mon.a (mon.0) 509 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-13T07:44:14.526 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:14 smithi089 bash[82331]: cluster 2024-04-13T07:44:13.237635+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e109: 8 total, 7 up, 8 in 2024-04-13T07:44:14.526 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:14 smithi089 bash[82331]: audit 2024-04-13T07:44:13.942747+0000 mon.a (mon.0) 511 : audit [DBG] from='client.? 172.21.15.89:0/653019870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:14.526 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:14 smithi089 bash[84777]: cluster 2024-04-13T07:44:13.116833+0000 mgr.y (mgr.34107) 270 : cluster [DBG] pgmap v207: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 255 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:14.526 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:14 smithi089 bash[84777]: cluster 2024-04-13T07:44:13.226339+0000 mon.a (mon.0) 509 : cluster [WRN] Health check failed: Reduced data availability: 5 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2024-04-13T07:44:14.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:14 smithi089 bash[84777]: cluster 2024-04-13T07:44:13.237635+0000 mon.a (mon.0) 510 : cluster [DBG] osdmap e109: 8 total, 7 up, 8 in 2024-04-13T07:44:14.527 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:14 smithi089 bash[84777]: audit 2024-04-13T07:44:13.942747+0000 mon.a (mon.0) 511 : audit [DBG] from='client.? 172.21.15.89:0/653019870' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:15.381 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:15.382 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:15.382 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:15.382 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:15.383 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:15.383 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:15.383 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:15.383 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.3.service: Deactivated successfully. 2024-04-13T07:44:15.384 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: Stopped Ceph osd.3 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:44:15.384 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:15.384 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: Started Ceph osd.3 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:44:15.384 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:15 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:16.212 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:44:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:44:15] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.43.0" 2024-04-13T07:44:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[82331]: cluster 2024-04-13T07:44:15.118017+0000 mgr.y (mgr.34107) 271 : cluster [DBG] pgmap v209: 73 pgs: 14 peering, 2 stale+active+clean, 10 active+undersized, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[82331]: audit 2024-04-13T07:44:15.427258+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[82331]: audit 2024-04-13T07:44:15.436954+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[82331]: audit 2024-04-13T07:44:15.453681+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:16.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[82331]: audit 2024-04-13T07:44:16.298940+0000 mon.a (mon.0) 515 : audit [DBG] from='client.? 172.21.15.89:0/104933886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[84777]: cluster 2024-04-13T07:44:15.118017+0000 mgr.y (mgr.34107) 271 : cluster [DBG] pgmap v209: 73 pgs: 14 peering, 2 stale+active+clean, 10 active+undersized, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:16.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[84777]: audit 2024-04-13T07:44:15.427258+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:16.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[84777]: audit 2024-04-13T07:44:15.436954+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:16.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[84777]: audit 2024-04-13T07:44:15.453681+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:16.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[84777]: audit 2024-04-13T07:44:16.298940+0000 mon.a (mon.0) 515 : audit [DBG] from='client.? 172.21.15.89:0/104933886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:16.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:16 smithi203 bash[68060]: cluster 2024-04-13T07:44:15.118017+0000 mgr.y (mgr.34107) 271 : cluster [DBG] pgmap v209: 73 pgs: 14 peering, 2 stale+active+clean, 10 active+undersized, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:16.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:16 smithi203 bash[68060]: audit 2024-04-13T07:44:15.427258+0000 mon.a (mon.0) 512 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:16.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:16 smithi203 bash[68060]: audit 2024-04-13T07:44:15.436954+0000 mon.a (mon.0) 513 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:16.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:16 smithi203 bash[68060]: audit 2024-04-13T07:44:15.453681+0000 mon.a (mon.0) 514 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:16.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:16 smithi203 bash[68060]: audit 2024-04-13T07:44:16.298940+0000 mon.a (mon.0) 515 : audit [DBG] from='client.? 172.21.15.89:0/104933886' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:17.054 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[105065]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-13T07:44:17.055 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[105065]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-3 --no-mon-config --dev /dev/mapper/vg_nvme-lv_1 2024-04-13T07:44:17.055 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[105065]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_1 2024-04-13T07:44:17.055 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[105065]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2024-04-13T07:44:17.055 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[105065]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_1 /var/lib/ceph/osd/ceph-3/block 2024-04-13T07:44:17.055 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[105065]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2024-04-13T07:44:17.055 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:16 smithi089 bash[105065]: --> ceph-volume raw activate successful for osd ID: 3 2024-04-13T07:44:17.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:17 smithi203 bash[68060]: cluster 2024-04-13T07:44:17.426863+0000 mon.a (mon.0) 516 : cluster [WRN] Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:44:17.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:17 smithi089 bash[84777]: cluster 2024-04-13T07:44:17.426863+0000 mon.a (mon.0) 516 : cluster [WRN] Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:44:17.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:17 smithi089 bash[82331]: cluster 2024-04-13T07:44:17.426863+0000 mon.a (mon.0) 516 : cluster [WRN] Health check failed: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2024-04-13T07:44:18.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:18 smithi203 bash[68060]: cluster 2024-04-13T07:44:17.118951+0000 mgr.y (mgr.34107) 272 : cluster [DBG] pgmap v210: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-13T07:44:18.798 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:18 smithi089 bash[105315]: debug 2024-04-13T07:44:18.404+0000 7f62aef3f700 1 -- 172.21.15.89:0/1943639246 <== mon.0 v2:172.21.15.89:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x55eadd25d680 con 0x55eadd29a800 2024-04-13T07:44:18.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:18 smithi089 bash[82331]: cluster 2024-04-13T07:44:17.118951+0000 mgr.y (mgr.34107) 272 : cluster [DBG] pgmap v210: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-13T07:44:18.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:18 smithi089 bash[84777]: cluster 2024-04-13T07:44:17.118951+0000 mgr.y (mgr.34107) 272 : cluster [DBG] pgmap v210: 73 pgs: 1 active+undersized+degraded, 14 peering, 16 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2024-04-13T07:44:19.298 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:18 smithi089 bash[105315]: debug 2024-04-13T07:44:18.960+0000 7f62b80d77c0 -1 Falling back to public interface 2024-04-13T07:44:19.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:19 smithi203 bash[68060]: audit 2024-04-13T07:44:18.647824+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/3653322056' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:19.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:19 smithi089 bash[82331]: audit 2024-04-13T07:44:18.647824+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/3653322056' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:19.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:19 smithi089 bash[84777]: audit 2024-04-13T07:44:18.647824+0000 mon.c (mon.2) 19 : audit [DBG] from='client.? 172.21.15.89:0/3653322056' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:20.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:20 smithi203 bash[68060]: cluster 2024-04-13T07:44:19.119967+0000 mgr.y (mgr.34107) 273 : cluster [DBG] pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:20.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:20 smithi203 bash[68060]: cluster 2024-04-13T07:44:19.481051+0000 mon.a (mon.0) 517 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-04-13T07:44:20.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:20 smithi089 bash[84777]: cluster 2024-04-13T07:44:19.119967+0000 mgr.y (mgr.34107) 273 : cluster [DBG] pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:20.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:20 smithi089 bash[84777]: cluster 2024-04-13T07:44:19.481051+0000 mon.a (mon.0) 517 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-04-13T07:44:20.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:20 smithi089 bash[82331]: cluster 2024-04-13T07:44:19.119967+0000 mgr.y (mgr.34107) 273 : cluster [DBG] pgmap v211: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:20.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:20 smithi089 bash[82331]: cluster 2024-04-13T07:44:19.481051+0000 mon.a (mon.0) 517 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 6 pgs peering) 2024-04-13T07:44:21.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:21 smithi203 bash[68060]: audit 2024-04-13T07:44:21.007890+0000 mon.a (mon.0) 518 : audit [DBG] from='client.? 172.21.15.89:0/1244088340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:21.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:21 smithi203 bash[68060]: audit 2024-04-13T07:44:21.242434+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:21 smithi203 bash[68060]: audit 2024-04-13T07:44:21.255091+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:21 smithi203 bash[68060]: audit 2024-04-13T07:44:21.443118+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:21 smithi203 bash[68060]: audit 2024-04-13T07:44:21.454358+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[84777]: audit 2024-04-13T07:44:21.007890+0000 mon.a (mon.0) 518 : audit [DBG] from='client.? 172.21.15.89:0/1244088340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:21.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[84777]: audit 2024-04-13T07:44:21.242434+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[84777]: audit 2024-04-13T07:44:21.255091+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[84777]: audit 2024-04-13T07:44:21.443118+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[84777]: audit 2024-04-13T07:44:21.454358+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[82331]: audit 2024-04-13T07:44:21.007890+0000 mon.a (mon.0) 518 : audit [DBG] from='client.? 172.21.15.89:0/1244088340' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:21.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[82331]: audit 2024-04-13T07:44:21.242434+0000 mon.a (mon.0) 519 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[82331]: audit 2024-04-13T07:44:21.255091+0000 mon.a (mon.0) 520 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[82331]: audit 2024-04-13T07:44:21.443118+0000 mon.a (mon.0) 521 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:21.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:21 smithi089 bash[82331]: audit 2024-04-13T07:44:21.454358+0000 mon.a (mon.0) 522 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:22.298 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[105315]: debug 2024-04-13T07:44:22.048+0000 7f62b80d77c0 -1 osd.3 107 log_to_monitors true 2024-04-13T07:44:22.764 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:44:22 smithi203 bash[60252]: ts=2024-04-13T07:44:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:44:22.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:22 smithi203 bash[68060]: cluster 2024-04-13T07:44:21.120487+0000 mgr.y (mgr.34107) 274 : cluster [DBG] pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:22.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:22 smithi203 bash[68060]: audit 2024-04-13T07:44:22.057963+0000 mon.a (mon.0) 523 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:44:22.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:22 smithi203 bash[68060]: audit 2024-04-13T07:44:22.254329+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:22.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:22 smithi203 bash[68060]: audit 2024-04-13T07:44:22.260095+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:22.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[84777]: cluster 2024-04-13T07:44:21.120487+0000 mgr.y (mgr.34107) 274 : cluster [DBG] pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:22.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[84777]: audit 2024-04-13T07:44:22.057963+0000 mon.a (mon.0) 523 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:44:22.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[84777]: audit 2024-04-13T07:44:22.254329+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:22.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[84777]: audit 2024-04-13T07:44:22.260095+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:22.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[82331]: cluster 2024-04-13T07:44:21.120487+0000 mgr.y (mgr.34107) 274 : cluster [DBG] pgmap v212: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[82331]: audit 2024-04-13T07:44:22.057963+0000 mon.a (mon.0) 523 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2024-04-13T07:44:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[82331]: audit 2024-04-13T07:44:22.254329+0000 mon.a (mon.0) 524 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:22 smithi089 bash[82331]: audit 2024-04-13T07:44:22.260095+0000 mon.a (mon.0) 525 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:23.513 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[105315]: debug 2024-04-13T07:44:23.052+0000 7f62a5527700 -1 osd.3 107 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-13T07:44:23.764 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:23 smithi203 bash[68060]: audit 2024-04-13T07:44:22.508406+0000 mon.a (mon.0) 526 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-13T07:44:23.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:23 smithi203 bash[68060]: cluster 2024-04-13T07:44:22.512834+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e110: 8 total, 7 up, 8 in 2024-04-13T07:44:23.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:23 smithi203 bash[68060]: audit 2024-04-13T07:44:22.513626+0000 mon.a (mon.0) 528 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:44:23.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:23 smithi203 bash[68060]: audit 2024-04-13T07:44:23.046159+0000 mon.a (mon.0) 529 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' 2024-04-13T07:44:23.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:23 smithi203 bash[68060]: audit 2024-04-13T07:44:23.364046+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/445765216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:23.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[82331]: audit 2024-04-13T07:44:22.508406+0000 mon.a (mon.0) 526 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-13T07:44:23.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[82331]: cluster 2024-04-13T07:44:22.512834+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e110: 8 total, 7 up, 8 in 2024-04-13T07:44:23.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[82331]: audit 2024-04-13T07:44:22.513626+0000 mon.a (mon.0) 528 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:44:23.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[82331]: audit 2024-04-13T07:44:23.046159+0000 mon.a (mon.0) 529 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' 2024-04-13T07:44:23.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[82331]: audit 2024-04-13T07:44:23.364046+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/445765216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:23.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[84777]: audit 2024-04-13T07:44:22.508406+0000 mon.a (mon.0) 526 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2024-04-13T07:44:23.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[84777]: cluster 2024-04-13T07:44:22.512834+0000 mon.a (mon.0) 527 : cluster [DBG] osdmap e110: 8 total, 7 up, 8 in 2024-04-13T07:44:23.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[84777]: audit 2024-04-13T07:44:22.513626+0000 mon.a (mon.0) 528 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi089", "root=default"]}]: dispatch 2024-04-13T07:44:23.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[84777]: audit 2024-04-13T07:44:23.046159+0000 mon.a (mon.0) 529 : audit [INF] from='osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897]' entity='osd.3' 2024-04-13T07:44:23.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:23 smithi089 bash[84777]: audit 2024-04-13T07:44:23.364046+0000 mon.c (mon.2) 20 : audit [DBG] from='client.? 172.21.15.89:0/445765216' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:24.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[82331]: debug 2024-04-13T07:44:24.040+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:24.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[84777]: cluster 2024-04-13T07:44:23.121506+0000 mgr.y (mgr.34107) 275 : cluster [DBG] pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:24.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[84777]: cluster 2024-04-13T07:44:24.045062+0000 mon.a (mon.0) 530 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[84777]: cluster 2024-04-13T07:44:24.045121+0000 mon.a (mon.0) 531 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[84777]: cluster 2024-04-13T07:44:24.055773+0000 mon.a (mon.0) 532 : cluster [INF] osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897] boot 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[84777]: cluster 2024-04-13T07:44:24.055941+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[84777]: audit 2024-04-13T07:44:24.059272+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[82331]: cluster 2024-04-13T07:44:23.121506+0000 mgr.y (mgr.34107) 275 : cluster [DBG] pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[82331]: cluster 2024-04-13T07:44:24.045062+0000 mon.a (mon.0) 530 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[82331]: cluster 2024-04-13T07:44:24.045121+0000 mon.a (mon.0) 531 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[82331]: cluster 2024-04-13T07:44:24.055773+0000 mon.a (mon.0) 532 : cluster [INF] osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897] boot 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[82331]: cluster 2024-04-13T07:44:24.055941+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-04-13T07:44:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:24 smithi089 bash[82331]: audit 2024-04-13T07:44:24.059272+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:44:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:24 smithi203 bash[68060]: cluster 2024-04-13T07:44:23.121506+0000 mgr.y (mgr.34107) 275 : cluster [DBG] pgmap v214: 73 pgs: 3 active+undersized+degraded, 28 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:24 smithi203 bash[68060]: cluster 2024-04-13T07:44:24.045062+0000 mon.a (mon.0) 530 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:24 smithi203 bash[68060]: cluster 2024-04-13T07:44:24.045121+0000 mon.a (mon.0) 531 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:44:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:24 smithi203 bash[68060]: cluster 2024-04-13T07:44:24.055773+0000 mon.a (mon.0) 532 : cluster [INF] osd.3 [v2:172.21.15.89:6826/367454897,v1:172.21.15.89:6827/367454897] boot 2024-04-13T07:44:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:24 smithi203 bash[68060]: cluster 2024-04-13T07:44:24.055941+0000 mon.a (mon.0) 533 : cluster [DBG] osdmap e111: 8 total, 8 up, 8 in 2024-04-13T07:44:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:24 smithi203 bash[68060]: audit 2024-04-13T07:44:24.059272+0000 mon.a (mon.0) 534 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2024-04-13T07:44:25.730 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev)": 4 2024-04-13T07:44:26.048 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:44:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:44:25] "GET /metrics HTTP/1.1" 200 34012 "" "Prometheus/2.43.0" 2024-04-13T07:44:26.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:26 smithi203 bash[68060]: cluster 2024-04-13T07:44:25.057567+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-04-13T07:44:26.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:26 smithi203 bash[68060]: cluster 2024-04-13T07:44:25.122213+0000 mgr.y (mgr.34107) 276 : cluster [DBG] pgmap v217: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:26.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:26 smithi203 bash[68060]: cluster 2024-04-13T07:44:25.633738+0000 mon.a (mon.0) 536 : cluster [WRN] Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:26.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:26 smithi203 bash[68060]: audit 2024-04-13T07:44:25.721952+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/3512730182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:26.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:26 smithi089 bash[84777]: cluster 2024-04-13T07:44:25.057567+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-04-13T07:44:26.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:26 smithi089 bash[84777]: cluster 2024-04-13T07:44:25.122213+0000 mgr.y (mgr.34107) 276 : cluster [DBG] pgmap v217: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:26.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:26 smithi089 bash[84777]: cluster 2024-04-13T07:44:25.633738+0000 mon.a (mon.0) 536 : cluster [WRN] Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:26.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:26 smithi089 bash[84777]: audit 2024-04-13T07:44:25.721952+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/3512730182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:26.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:26 smithi089 bash[82331]: cluster 2024-04-13T07:44:25.057567+0000 mon.a (mon.0) 535 : cluster [DBG] osdmap e112: 8 total, 8 up, 8 in 2024-04-13T07:44:26.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:26 smithi089 bash[82331]: cluster 2024-04-13T07:44:25.122213+0000 mgr.y (mgr.34107) 276 : cluster [DBG] pgmap v217: 73 pgs: 5 peering, 3 active+undersized+degraded, 23 active+undersized, 42 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:26.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:26 smithi089 bash[82331]: cluster 2024-04-13T07:44:25.633738+0000 mon.a (mon.0) 536 : cluster [WRN] Health check update: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:26.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:26 smithi089 bash[82331]: audit 2024-04-13T07:44:25.721952+0000 mon.c (mon.2) 21 : audit [DBG] from='client.? 172.21.15.89:0/3512730182' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2024-04-13T07:44:26.589 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch upgrade pause' 2024-04-13T07:44:28.401 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:28 smithi089 bash[84777]: audit 2024-04-13T07:44:27.080039+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:28.401 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:28 smithi089 bash[84777]: audit 2024-04-13T07:44:27.082381+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:28.401 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:28 smithi089 bash[84777]: cluster 2024-04-13T07:44:27.123060+0000 mgr.y (mgr.34107) 277 : cluster [DBG] pgmap v218: 73 pgs: 5 peering, 3 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:28.401 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:28 smithi089 bash[82331]: audit 2024-04-13T07:44:27.080039+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:28.401 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:28 smithi089 bash[82331]: audit 2024-04-13T07:44:27.082381+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:28.401 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:28 smithi089 bash[82331]: cluster 2024-04-13T07:44:27.123060+0000 mgr.y (mgr.34107) 277 : cluster [DBG] pgmap v218: 73 pgs: 5 peering, 3 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:28.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:28 smithi203 bash[68060]: audit 2024-04-13T07:44:27.080039+0000 mon.a (mon.0) 537 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:28.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:28 smithi203 bash[68060]: audit 2024-04-13T07:44:27.082381+0000 mon.a (mon.0) 538 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:28.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:28 smithi203 bash[68060]: cluster 2024-04-13T07:44:27.123060+0000 mgr.y (mgr.34107) 277 : cluster [DBG] pgmap v218: 73 pgs: 5 peering, 3 active+undersized+degraded, 18 active+undersized, 47 active+clean; 4.6 MiB data, 256 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2024-04-13T07:44:29.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:29 smithi203 bash[68060]: cluster 2024-04-13T07:44:29.175598+0000 mon.a (mon.0) 539 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-04-13T07:44:29.544 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:29 smithi089 bash[82331]: cluster 2024-04-13T07:44:29.175598+0000 mon.a (mon.0) 539 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-04-13T07:44:29.545 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:29 smithi089 bash[84777]: cluster 2024-04-13T07:44:29.175598+0000 mon.a (mon.0) 539 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded) 2024-04-13T07:44:30.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:30 smithi203 bash[68060]: cluster 2024-04-13T07:44:29.124013+0000 mgr.y (mgr.34107) 278 : cluster [DBG] pgmap v219: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:30.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:30 smithi089 bash[82331]: cluster 2024-04-13T07:44:29.124013+0000 mgr.y (mgr.34107) 278 : cluster [DBG] pgmap v219: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:30.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:30 smithi089 bash[84777]: cluster 2024-04-13T07:44:29.124013+0000 mgr.y (mgr.34107) 278 : cluster [DBG] pgmap v219: 73 pgs: 5 peering, 68 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:31.854 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:44:32.498 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[60252]: ts=2024-04-13T07:44:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:44:32.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: cluster 2024-04-13T07:44:31.124890+0000 mgr.y (mgr.34107) 279 : cluster [DBG] pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:32.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.506633+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.518705+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.521412+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:32.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.523245+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:44:32.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.536335+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.633080+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:32.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.638080+0000 mon.a (mon.0) 546 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.642100+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.644996+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.648910+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-13T07:44:32.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: audit 2024-04-13T07:44:31.649338+0000 mgr.y (mgr.34107) 280 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-13T07:44:32.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:32 smithi203 bash[68060]: cephadm 2024-04-13T07:44:31.651403+0000 mgr.y (mgr.34107) 281 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: cluster 2024-04-13T07:44:31.124890+0000 mgr.y (mgr.34107) 279 : cluster [DBG] pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.506633+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.518705+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.521412+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.523245+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.536335+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.633080+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.638080+0000 mon.a (mon.0) 546 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.642100+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.644996+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.648910+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: audit 2024-04-13T07:44:31.649338+0000 mgr.y (mgr.34107) 280 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-13T07:44:32.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[82331]: cephadm 2024-04-13T07:44:31.651403+0000 mgr.y (mgr.34107) 281 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: cluster 2024-04-13T07:44:31.124890+0000 mgr.y (mgr.34107) 279 : cluster [DBG] pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.506633+0000 mon.a (mon.0) 540 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.518705+0000 mon.a (mon.0) 541 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.521412+0000 mon.a (mon.0) 542 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.523245+0000 mon.a (mon.0) 543 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.536335+0000 mon.a (mon.0) 544 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.633080+0000 mon.a (mon.0) 545 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.638080+0000 mon.a (mon.0) 546 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.642100+0000 mon.a (mon.0) 547 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.644996+0000 mon.a (mon.0) 548 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.648910+0000 mon.a (mon.0) 549 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: audit 2024-04-13T07:44:31.649338+0000 mgr.y (mgr.34107) 280 : audit [DBG] from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2024-04-13T07:44:32.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:32 smithi089 bash[84777]: cephadm 2024-04-13T07:44:31.651403+0000 mgr.y (mgr.34107) 281 : cephadm [INF] Upgrade: osd.4 is safe to restart 2024-04-13T07:44:34.457 INFO:teuthology.orchestra.run.smithi089.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:44:34.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:34 smithi203 bash[68060]: cluster 2024-04-13T07:44:33.125893+0000 mgr.y (mgr.34107) 282 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:34.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:34 smithi203 bash[68060]: audit 2024-04-13T07:44:34.378340+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:34.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:34 smithi203 bash[68060]: audit 2024-04-13T07:44:34.382270+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:44:34.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:34 smithi203 bash[68060]: audit 2024-04-13T07:44:34.383345+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:34.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:34 smithi203 bash[68060]: audit 2024-04-13T07:44:34.463549+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:34.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[82331]: cluster 2024-04-13T07:44:33.125893+0000 mgr.y (mgr.34107) 282 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:34.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[82331]: audit 2024-04-13T07:44:34.378340+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:34.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[82331]: audit 2024-04-13T07:44:34.382270+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:44:34.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[82331]: audit 2024-04-13T07:44:34.383345+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:34.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[82331]: audit 2024-04-13T07:44:34.463549+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:34.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[84777]: cluster 2024-04-13T07:44:33.125893+0000 mgr.y (mgr.34107) 282 : cluster [DBG] pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:34.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[84777]: audit 2024-04-13T07:44:34.378340+0000 mon.a (mon.0) 550 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:34.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[84777]: audit 2024-04-13T07:44:34.382270+0000 mon.a (mon.0) 551 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2024-04-13T07:44:34.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[84777]: audit 2024-04-13T07:44:34.383345+0000 mon.a (mon.0) 552 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:44:34.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:34 smithi089 bash[84777]: audit 2024-04-13T07:44:34.463549+0000 mon.a (mon.0) 553 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:35.166 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch ps' 2024-04-13T07:44:35.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:35 smithi203 bash[68060]: cephadm 2024-04-13T07:44:34.371738+0000 mgr.y (mgr.34107) 283 : cephadm [INF] Upgrade: Updating osd.4 2024-04-13T07:44:35.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:35 smithi203 bash[68060]: cephadm 2024-04-13T07:44:34.387638+0000 mgr.y (mgr.34107) 284 : cephadm [INF] Deploying daemon osd.4 on smithi203 2024-04-13T07:44:35.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:35 smithi203 bash[68060]: audit 2024-04-13T07:44:34.458228+0000 mgr.y (mgr.34107) 285 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:35.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:35 smithi203 bash[68060]: cephadm 2024-04-13T07:44:34.458814+0000 mgr.y (mgr.34107) 286 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:44:35.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[82331]: cephadm 2024-04-13T07:44:34.371738+0000 mgr.y (mgr.34107) 283 : cephadm [INF] Upgrade: Updating osd.4 2024-04-13T07:44:35.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[82331]: cephadm 2024-04-13T07:44:34.387638+0000 mgr.y (mgr.34107) 284 : cephadm [INF] Deploying daemon osd.4 on smithi203 2024-04-13T07:44:35.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[82331]: audit 2024-04-13T07:44:34.458228+0000 mgr.y (mgr.34107) 285 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:35.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[82331]: cephadm 2024-04-13T07:44:34.458814+0000 mgr.y (mgr.34107) 286 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:44:35.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[84777]: cephadm 2024-04-13T07:44:34.371738+0000 mgr.y (mgr.34107) 283 : cephadm [INF] Upgrade: Updating osd.4 2024-04-13T07:44:35.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[84777]: cephadm 2024-04-13T07:44:34.387638+0000 mgr.y (mgr.34107) 284 : cephadm [INF] Deploying daemon osd.4 on smithi203 2024-04-13T07:44:35.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[84777]: audit 2024-04-13T07:44:34.458228+0000 mgr.y (mgr.34107) 285 : audit [DBG] from='client.44324 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:35.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[84777]: cephadm 2024-04-13T07:44:34.458814+0000 mgr.y (mgr.34107) 286 : cephadm [INF] Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:6fb717258dc239385e7a35961ace9777596e0812fffb5512947cc85619c020de 2024-04-13T07:44:36.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:44:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:44:35] "GET /metrics HTTP/1.1" 200 33859 "" "Prometheus/2.43.0" 2024-04-13T07:44:36.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:36 smithi203 bash[68060]: cluster 2024-04-13T07:44:35.126657+0000 mgr.y (mgr.34107) 287 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:36.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:36 smithi089 bash[82331]: cluster 2024-04-13T07:44:35.126657+0000 mgr.y (mgr.34107) 287 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:36.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:36 smithi089 bash[84777]: cluster 2024-04-13T07:44:35.126657+0000 mgr.y (mgr.34107) 287 : cluster [DBG] pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:38 smithi203 bash[68060]: cluster 2024-04-13T07:44:37.127417+0000 mgr.y (mgr.34107) 288 : cluster [DBG] pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:38.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:38 smithi203 bash[68060]: audit 2024-04-13T07:44:37.232561+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:38.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:38 smithi089 bash[84777]: cluster 2024-04-13T07:44:37.127417+0000 mgr.y (mgr.34107) 288 : cluster [DBG] pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:38.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:38 smithi089 bash[84777]: audit 2024-04-13T07:44:37.232561+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:38.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:38 smithi089 bash[82331]: cluster 2024-04-13T07:44:37.127417+0000 mgr.y (mgr.34107) 288 : cluster [DBG] pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:38.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:38 smithi089 bash[82331]: audit 2024-04-13T07:44:37.232561+0000 mon.a (mon.0) 554 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:39.294 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.294 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.295 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.295 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.295 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.296 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.296 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.296 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.296 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: Stopping Ceph osd.4 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:44:39.296 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:44:39 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:39.765 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:39 smithi203 bash[27023]: debug 2024-04-13T07:44:39.419+0000 7f101f098700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:44:39.765 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:39 smithi203 bash[27023]: debug 2024-04-13T07:44:39.419+0000 7f101f098700 -1 osd.4 112 *** Got signal Terminated *** 2024-04-13T07:44:39.765 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:39 smithi203 bash[27023]: debug 2024-04-13T07:44:39.419+0000 7f101f098700 -1 osd.4 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:44:39.902 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:44:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:40 smithi203 bash[68060]: cluster 2024-04-13T07:44:39.128340+0000 mgr.y (mgr.34107) 289 : cluster [DBG] pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:40.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:40 smithi203 bash[68060]: cluster 2024-04-13T07:44:39.425830+0000 mon.a (mon.0) 555 : cluster [INF] osd.4 marked itself down and dead 2024-04-13T07:44:40.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:40 smithi089 bash[84777]: cluster 2024-04-13T07:44:39.128340+0000 mgr.y (mgr.34107) 289 : cluster [DBG] pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:40.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:40 smithi089 bash[84777]: cluster 2024-04-13T07:44:39.425830+0000 mon.a (mon.0) 555 : cluster [INF] osd.4 marked itself down and dead 2024-04-13T07:44:40.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:40 smithi089 bash[82331]: debug 2024-04-13T07:44:40.227+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:40.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:40 smithi089 bash[82331]: cluster 2024-04-13T07:44:39.128340+0000 mgr.y (mgr.34107) 289 : cluster [DBG] pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:40.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:40 smithi089 bash[82331]: cluster 2024-04-13T07:44:39.425830+0000 mon.a (mon.0) 555 : cluster [INF] osd.4 marked itself down and dead 2024-04-13T07:44:41.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:41 smithi203 bash[68060]: cluster 2024-04-13T07:44:40.233833+0000 mon.a (mon.0) 556 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:44:41.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:41 smithi203 bash[68060]: cluster 2024-04-13T07:44:40.233896+0000 mon.a (mon.0) 557 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:41.265 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:41 smithi203 bash[68060]: cluster 2024-04-13T07:44:40.254916+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e113: 8 total, 7 up, 8 in 2024-04-13T07:44:41.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:41 smithi089 bash[82331]: cluster 2024-04-13T07:44:40.233833+0000 mon.a (mon.0) 556 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:44:41.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:41 smithi089 bash[82331]: cluster 2024-04-13T07:44:40.233896+0000 mon.a (mon.0) 557 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:41.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:41 smithi089 bash[82331]: cluster 2024-04-13T07:44:40.254916+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e113: 8 total, 7 up, 8 in 2024-04-13T07:44:41.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:41 smithi089 bash[84777]: cluster 2024-04-13T07:44:40.233833+0000 mon.a (mon.0) 556 : cluster [WRN] Health check failed: 1 osds down (OSD_DOWN) 2024-04-13T07:44:41.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:41 smithi089 bash[84777]: cluster 2024-04-13T07:44:40.233896+0000 mon.a (mon.0) 557 : cluster [ERR] Health check update: 7 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:41.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:41 smithi089 bash[84777]: cluster 2024-04-13T07:44:40.254916+0000 mon.a (mon.0) 558 : cluster [DBG] osdmap e113: 8 total, 7 up, 8 in 2024-04-13T07:44:41.764 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:41 smithi203 bash[73280]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-4 2024-04-13T07:44:41.765 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:41 smithi203 bash[73345]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-4 2024-04-13T07:44:42.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:44:42 smithi203 bash[60252]: ts=2024-04-13T07:44:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:44:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:42 smithi203 bash[68060]: cluster 2024-04-13T07:44:41.129125+0000 mgr.y (mgr.34107) 290 : cluster [DBG] pgmap v226: 73 pgs: 5 peering, 12 stale+active+clean, 56 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:42 smithi203 bash[68060]: cluster 2024-04-13T07:44:41.244690+0000 mon.a (mon.0) 559 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-13T07:44:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:42 smithi203 bash[68060]: cluster 2024-04-13T07:44:41.260421+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e114: 8 total, 7 up, 8 in 2024-04-13T07:44:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:42 smithi203 bash[68060]: audit 2024-04-13T07:44:42.079939+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:42.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:42 smithi203 bash[68060]: audit 2024-04-13T07:44:42.082182+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:42.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[82331]: cluster 2024-04-13T07:44:41.129125+0000 mgr.y (mgr.34107) 290 : cluster [DBG] pgmap v226: 73 pgs: 5 peering, 12 stale+active+clean, 56 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[82331]: cluster 2024-04-13T07:44:41.244690+0000 mon.a (mon.0) 559 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[82331]: cluster 2024-04-13T07:44:41.260421+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e114: 8 total, 7 up, 8 in 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[82331]: audit 2024-04-13T07:44:42.079939+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[82331]: audit 2024-04-13T07:44:42.082182+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[84777]: cluster 2024-04-13T07:44:41.129125+0000 mgr.y (mgr.34107) 290 : cluster [DBG] pgmap v226: 73 pgs: 5 peering, 12 stale+active+clean, 56 active+clean; 4.6 MiB data, 260 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[84777]: cluster 2024-04-13T07:44:41.244690+0000 mon.a (mon.0) 559 : cluster [WRN] Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[84777]: cluster 2024-04-13T07:44:41.260421+0000 mon.a (mon.0) 560 : cluster [DBG] osdmap e114: 8 total, 7 up, 8 in 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[84777]: audit 2024-04-13T07:44:42.079939+0000 mon.a (mon.0) 561 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:42.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:42 smithi089 bash[84777]: audit 2024-04-13T07:44:42.082182+0000 mon.a (mon.0) 562 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:42.711 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (14m) 21s ago 14m 14.2M - 0.25.0 c8568f914cd2 d39c47986efe 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi203 *:3000 running (14m) 21s ago 14m 78.1M - 9.4.7 954c08fa6188 4496ec520de5 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.pccnvq smithi089 running (12m) 21s ago 12m 15.5M - 18.2.2-1217-g7d36322c 6ecd1226aa2f 07167bfc6e5f 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi203.sjspdy smithi203 running (12m) 21s ago 12m 17.9M - 18.2.2-1217-g7d36322c 6ecd1226aa2f e100a52f8447 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi203 *:8443,9283,8765 running (7m) 21s ago 22m 439M - 19.0.0-2905-g5f22a497 c4b45bf0c946 54a5a6c61bc7 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (8m) 21s ago 25m 498M - 19.0.0-2905-g5f22a497 c4b45bf0c946 f20e719fbf68 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (6m) 21s ago 25m 52.0M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 d2e86a9f9ec2 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi203 running (4m) 21s ago 23m 36.1M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 fb3dac884f39 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (5m) 21s ago 23m 38.4M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 0345f038f9be 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (14m) 21s ago 14m 9228k - 1.5.0 0da6a335fe13 04462903d931 2024-04-13T07:44:42.712 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi203 *:9100 running (14m) 21s ago 14m 8867k - 1.5.0 0da6a335fe13 a79084a2b37c 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (3m) 21s ago 22m 47.9M 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 dffd1ec4051b 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (78s) 21s ago 21m 50.5M 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 a3e8d9018ab2 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (53s) 21s ago 20m 40.8M 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 c58448891ad9 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (25s) 21s ago 19m - 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 81871fb8c71f 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi203 running (18m) 21s ago 18m 57.5M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 28af48f1a163 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi203 running (17m) 21s ago 17m 50.4M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f afe79fa4d7d0 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi203 running (17m) 21s ago 17m 49.8M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 8913a43d05b8 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi203 running (16m) 21s ago 16m 64.8M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 01ed33979f80 2024-04-13T07:44:42.713 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi203 *:9095 running (7m) 21s ago 15m 50.4M - 2.43.0 a07b618ecd1d 50a6268920e0 2024-04-13T07:44:43.437 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph orch ps' 2024-04-13T07:44:43.514 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:43 smithi203 bash[68060]: audit 2024-04-13T07:44:42.690361+0000 mgr.y (mgr.34107) 291 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:43.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:43 smithi089 bash[84777]: audit 2024-04-13T07:44:42.690361+0000 mgr.y (mgr.34107) 291 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:43.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:43 smithi089 bash[82331]: audit 2024-04-13T07:44:42.690361+0000 mgr.y (mgr.34107) 291 : audit [DBG] from='client.34398 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:44.068 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.069 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.069 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.069 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.4.service: Deactivated successfully. 2024-04-13T07:44:44.069 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: Stopped Ceph osd.4 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:44:44.069 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.070 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.070 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.070 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.070 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.071 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:44:43 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:44:44.515 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:44 smithi203 systemd[1]: Started Ceph osd.4 for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:44:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:44 smithi203 bash[68060]: cluster 2024-04-13T07:44:43.130234+0000 mgr.y (mgr.34107) 292 : cluster [DBG] pgmap v228: 73 pgs: 8 active+undersized+degraded, 5 peering, 4 stale+active+clean, 12 active+undersized, 44 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:44:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:44 smithi203 bash[68060]: cluster 2024-04-13T07:44:43.239343+0000 mon.a (mon.0) 563 : cluster [WRN] Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:44 smithi203 bash[68060]: audit 2024-04-13T07:44:44.126002+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:44.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:44 smithi203 bash[68060]: audit 2024-04-13T07:44:44.137767+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:44.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:44 smithi203 bash[68060]: audit 2024-04-13T07:44:44.178608+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:44.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[82331]: cluster 2024-04-13T07:44:43.130234+0000 mgr.y (mgr.34107) 292 : cluster [DBG] pgmap v228: 73 pgs: 8 active+undersized+degraded, 5 peering, 4 stale+active+clean, 12 active+undersized, 44 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:44:44.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[82331]: cluster 2024-04-13T07:44:43.239343+0000 mon.a (mon.0) 563 : cluster [WRN] Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:44.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[82331]: audit 2024-04-13T07:44:44.126002+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:44.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[82331]: audit 2024-04-13T07:44:44.137767+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:44.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[82331]: audit 2024-04-13T07:44:44.178608+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:44.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[84777]: cluster 2024-04-13T07:44:43.130234+0000 mgr.y (mgr.34107) 292 : cluster [DBG] pgmap v228: 73 pgs: 8 active+undersized+degraded, 5 peering, 4 stale+active+clean, 12 active+undersized, 44 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:44:44.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[84777]: cluster 2024-04-13T07:44:43.239343+0000 mon.a (mon.0) 563 : cluster [WRN] Health check failed: Degraded data redundancy: 13/84 objects degraded (15.476%), 8 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:44.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[84777]: audit 2024-04-13T07:44:44.126002+0000 mon.a (mon.0) 564 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:44.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[84777]: audit 2024-04-13T07:44:44.137767+0000 mon.a (mon.0) 565 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:44.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:44 smithi089 bash[84777]: audit 2024-04-13T07:44:44.178608+0000 mon.a (mon.0) 566 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:44:46.015 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:45 smithi203 bash[73501]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-13T07:44:46.015 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:45 smithi203 bash[73501]: Running command: /usr/bin/ceph-bluestore-tool prime-osd-dir --path /var/lib/ceph/osd/ceph-4 --no-mon-config --dev /dev/mapper/vg_nvme-lv_4 2024-04-13T07:44:46.015 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:45 smithi203 bash[73501]: Running command: /usr/bin/chown -h ceph:ceph /dev/mapper/vg_nvme-lv_4 2024-04-13T07:44:46.015 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:45 smithi203 bash[73501]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2024-04-13T07:44:46.015 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:45 smithi203 bash[73501]: Running command: /usr/bin/ln -s /dev/mapper/vg_nvme-lv_4 /var/lib/ceph/osd/ceph-4/block 2024-04-13T07:44:46.015 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:45 smithi203 bash[73501]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-4 2024-04-13T07:44:46.016 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:45 smithi203 bash[73501]: --> ceph-volume raw activate successful for osd ID: 4 2024-04-13T07:44:46.251 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:44:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:44:45] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:44:46.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:46 smithi203 bash[68060]: cluster 2024-04-13T07:44:45.131221+0000 mgr.y (mgr.34107) 293 : cluster [DBG] pgmap v229: 73 pgs: 8 active+undersized+degraded, 5 peering, 20 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:44:46.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:46 smithi089 bash[82331]: cluster 2024-04-13T07:44:45.131221+0000 mgr.y (mgr.34107) 293 : cluster [DBG] pgmap v229: 73 pgs: 8 active+undersized+degraded, 5 peering, 20 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:44:46.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:46 smithi089 bash[84777]: cluster 2024-04-13T07:44:45.131221+0000 mgr.y (mgr.34107) 293 : cluster [DBG] pgmap v229: 73 pgs: 8 active+undersized+degraded, 5 peering, 20 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 13/84 objects degraded (15.476%) 2024-04-13T07:44:47.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:47 smithi203 bash[68060]: cluster 2024-04-13T07:44:47.257453+0000 mon.a (mon.0) 567 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-13T07:44:47.765 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:47 smithi203 bash[73720]: debug 2024-04-13T07:44:47.539+0000 7f2ab87bb700 1 -- 172.21.15.203:0/3691428427 <== mon.1 v2:172.21.15.203:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) ==== 194+0+0 (secure 0 0 0) 0x563e91d77680 con 0x563e91db4800 2024-04-13T07:44:47.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:47 smithi089 bash[82331]: cluster 2024-04-13T07:44:47.257453+0000 mon.a (mon.0) 567 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-13T07:44:47.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:47 smithi089 bash[84777]: cluster 2024-04-13T07:44:47.257453+0000 mon.a (mon.0) 567 : cluster [INF] Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2024-04-13T07:44:48.172 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:44:48.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:48 smithi203 bash[68060]: cluster 2024-04-13T07:44:47.132069+0000 mgr.y (mgr.34107) 294 : cluster [DBG] pgmap v230: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:48.515 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:48 smithi203 bash[73720]: debug 2024-04-13T07:44:48.075+0000 7f2ac19537c0 -1 Falling back to public interface 2024-04-13T07:44:48.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:48 smithi089 bash[84777]: cluster 2024-04-13T07:44:47.132069+0000 mgr.y (mgr.34107) 294 : cluster [DBG] pgmap v230: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:48.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:48 smithi089 bash[82331]: cluster 2024-04-13T07:44:47.132069+0000 mgr.y (mgr.34107) 294 : cluster [DBG] pgmap v230: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:50 smithi203 bash[68060]: cluster 2024-04-13T07:44:49.132687+0000 mgr.y (mgr.34107) 295 : cluster [DBG] pgmap v231: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:50 smithi203 bash[68060]: audit 2024-04-13T07:44:50.174121+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:50.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:50 smithi203 bash[68060]: audit 2024-04-13T07:44:50.181210+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:50.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:50 smithi089 bash[84777]: cluster 2024-04-13T07:44:49.132687+0000 mgr.y (mgr.34107) 295 : cluster [DBG] pgmap v231: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:50.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:50 smithi089 bash[84777]: audit 2024-04-13T07:44:50.174121+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:50.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:50 smithi089 bash[84777]: audit 2024-04-13T07:44:50.181210+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:50.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:50 smithi089 bash[82331]: cluster 2024-04-13T07:44:49.132687+0000 mgr.y (mgr.34107) 295 : cluster [DBG] pgmap v231: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:50.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:50 smithi089 bash[82331]: audit 2024-04-13T07:44:50.174121+0000 mon.a (mon.0) 568 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:50.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:50 smithi089 bash[82331]: audit 2024-04-13T07:44:50.181210+0000 mon.a (mon.0) 569 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:alertmanager.a smithi089 *:9093,9094 running (14m) 29s ago 14m 14.2M - 0.25.0 c8568f914cd2 d39c47986efe 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:grafana.a smithi203 *:3000 running (14m) 0s ago 14m 78.2M - 9.4.7 954c08fa6188 4496ec520de5 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi089.pccnvq smithi089 running (12m) 29s ago 12m 15.5M - 18.2.2-1217-g7d36322c 6ecd1226aa2f 07167bfc6e5f 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:mds.foo.smithi203.sjspdy smithi203 running (12m) 0s ago 12m 18.0M - 18.2.2-1217-g7d36322c 6ecd1226aa2f e100a52f8447 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.x smithi203 *:8443,9283,8765 running (7m) 0s ago 22m 439M - 19.0.0-2905-g5f22a497 c4b45bf0c946 54a5a6c61bc7 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:mgr.y smithi089 *:8443,9283,8765 running (8m) 29s ago 25m 498M - 19.0.0-2905-g5f22a497 c4b45bf0c946 f20e719fbf68 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:mon.a smithi089 running (7m) 29s ago 25m 52.0M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 d2e86a9f9ec2 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:mon.b smithi203 running (4m) 0s ago 23m 37.0M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 fb3dac884f39 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:mon.c smithi089 running (5m) 29s ago 23m 38.4M 2048M 19.0.0-2905-g5f22a497 c4b45bf0c946 0345f038f9be 2024-04-13T07:44:50.850 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.a smithi089 *:9100 running (15m) 29s ago 15m 9228k - 1.5.0 0da6a335fe13 04462903d931 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:node-exporter.b smithi203 *:9100 running (14m) 0s ago 15m 8939k - 1.5.0 0da6a335fe13 a79084a2b37c 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:osd.0 smithi089 running (3m) 29s ago 22m 47.9M 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 dffd1ec4051b 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:osd.1 smithi089 running (87s) 29s ago 21m 50.5M 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 a3e8d9018ab2 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:osd.2 smithi089 running (61s) 29s ago 20m 40.8M 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 c58448891ad9 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:osd.3 smithi089 running (33s) 29s ago 19m - 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 81871fb8c71f 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:osd.4 smithi203 running (4s) 0s ago 18m - 2524M 19.0.0-2905-g5f22a497 c4b45bf0c946 f348b843148f 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:osd.5 smithi203 running (18m) 0s ago 18m 51.2M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f afe79fa4d7d0 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:osd.6 smithi203 running (17m) 0s ago 17m 49.7M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 8913a43d05b8 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:osd.7 smithi203 running (16m) 0s ago 16m 65.7M 2524M 18.2.2-1217-g7d36322c 6ecd1226aa2f 01ed33979f80 2024-04-13T07:44:50.851 INFO:teuthology.orchestra.run.smithi089.stdout:prometheus.a smithi203 *:9095 running (7m) 0s ago 15m 50.7M - 2.43.0 a07b618ecd1d 50a6268920e0 2024-04-13T07:44:51.377 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:51 smithi203 bash[73720]: debug 2024-04-13T07:44:51.127+0000 7f2ac19537c0 -1 osd.4 112 log_to_monitors true 2024-04-13T07:44:51.377 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:51 smithi203 bash[68060]: cluster 2024-04-13T07:44:50.638900+0000 mon.a (mon.0) 570 : cluster [WRN] Health check update: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:51.377 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:51 smithi203 bash[68060]: audit 2024-04-13T07:44:51.133432+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:44:51.377 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:51 smithi203 bash[68060]: audit 2024-04-13T07:44:51.133555+0000 mon.a (mon.0) 571 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:44:51.377 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:51 smithi203 bash[68060]: audit 2024-04-13T07:44:51.290366+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:51.377 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:51 smithi203 bash[68060]: audit 2024-04-13T07:44:51.303857+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:51.598 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[82331]: cluster 2024-04-13T07:44:50.638900+0000 mon.a (mon.0) 570 : cluster [WRN] Health check update: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:51.598 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[82331]: audit 2024-04-13T07:44:51.133432+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:44:51.598 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[82331]: audit 2024-04-13T07:44:51.133555+0000 mon.a (mon.0) 571 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:44:51.598 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[82331]: audit 2024-04-13T07:44:51.290366+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:51.598 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[82331]: audit 2024-04-13T07:44:51.303857+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:51.598 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[84777]: cluster 2024-04-13T07:44:50.638900+0000 mon.a (mon.0) 570 : cluster [WRN] Health check update: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:51.598 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[84777]: audit 2024-04-13T07:44:51.133432+0000 mon.b (mon.1) 3 : audit [INF] from='osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:44:51.598 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[84777]: audit 2024-04-13T07:44:51.133555+0000 mon.a (mon.0) 571 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2024-04-13T07:44:51.599 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[84777]: audit 2024-04-13T07:44:51.290366+0000 mon.a (mon.0) 572 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:51.599 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:51 smithi089 bash[84777]: audit 2024-04-13T07:44:51.303857+0000 mon.a (mon.0) 573 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:51.599 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -e sha1=5f22a4974d6492df1d519913f5251731f043c582 -- bash -c 'ceph versions' 2024-04-13T07:44:51.765 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:44:51 smithi203 bash[73720]: debug 2024-04-13T07:44:51.383+0000 7f2aaeda3700 -1 osd.4 112 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2024-04-13T07:44:52.765 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:44:52 smithi203 bash[60252]: ts=2024-04-13T07:44:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:44:52.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:52 smithi203 bash[68060]: audit 2024-04-13T07:44:50.814504+0000 mgr.y (mgr.34107) 296 : audit [DBG] from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:52.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:52 smithi203 bash[68060]: cluster 2024-04-13T07:44:51.133684+0000 mgr.y (mgr.34107) 297 : cluster [DBG] pgmap v232: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:52.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:52 smithi203 bash[68060]: audit 2024-04-13T07:44:51.328086+0000 mon.a (mon.0) 574 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-13T07:44:52.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:52 smithi203 bash[68060]: audit 2024-04-13T07:44:51.335383+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:44:52.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:52 smithi203 bash[68060]: cluster 2024-04-13T07:44:51.337714+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e115: 8 total, 7 up, 8 in 2024-04-13T07:44:52.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:52 smithi203 bash[68060]: audit 2024-04-13T07:44:51.338964+0000 mon.a (mon.0) 576 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[82331]: debug 2024-04-13T07:44:52.323+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[82331]: audit 2024-04-13T07:44:50.814504+0000 mgr.y (mgr.34107) 296 : audit [DBG] from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[82331]: cluster 2024-04-13T07:44:51.133684+0000 mgr.y (mgr.34107) 297 : cluster [DBG] pgmap v232: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[82331]: audit 2024-04-13T07:44:51.328086+0000 mon.a (mon.0) 574 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[82331]: audit 2024-04-13T07:44:51.335383+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[82331]: cluster 2024-04-13T07:44:51.337714+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e115: 8 total, 7 up, 8 in 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[82331]: audit 2024-04-13T07:44:51.338964+0000 mon.a (mon.0) 576 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[84777]: audit 2024-04-13T07:44:50.814504+0000 mgr.y (mgr.34107) 296 : audit [DBG] from='client.34404 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[84777]: cluster 2024-04-13T07:44:51.133684+0000 mgr.y (mgr.34107) 297 : cluster [DBG] pgmap v232: 73 pgs: 9 active+undersized+degraded, 24 active+undersized, 40 active+clean; 4.6 MiB data, 261 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[84777]: audit 2024-04-13T07:44:51.328086+0000 mon.a (mon.0) 574 : audit [INF] from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[84777]: audit 2024-04-13T07:44:51.335383+0000 mon.b (mon.1) 4 : audit [INF] from='osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:44:52.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[84777]: cluster 2024-04-13T07:44:51.337714+0000 mon.a (mon.0) 575 : cluster [DBG] osdmap e115: 8 total, 7 up, 8 in 2024-04-13T07:44:52.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:52 smithi089 bash[84777]: audit 2024-04-13T07:44:51.338964+0000 mon.a (mon.0) 576 : audit [INF] from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi203", "root=default"]}]: dispatch 2024-04-13T07:44:53.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:53 smithi203 bash[68060]: cluster 2024-04-13T07:44:52.329005+0000 mon.a (mon.0) 577 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:53.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:53 smithi203 bash[68060]: cluster 2024-04-13T07:44:52.329060+0000 mon.a (mon.0) 578 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:44:53.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:53 smithi203 bash[68060]: cluster 2024-04-13T07:44:52.346665+0000 mon.a (mon.0) 579 : cluster [INF] osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489] boot 2024-04-13T07:44:53.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:53 smithi203 bash[68060]: cluster 2024-04-13T07:44:52.346857+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-04-13T07:44:53.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:53 smithi203 bash[68060]: audit 2024-04-13T07:44:52.349605+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:44:53.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:53 smithi203 bash[68060]: cluster 2024-04-13T07:44:53.341187+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-04-13T07:44:53.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[82331]: cluster 2024-04-13T07:44:52.329005+0000 mon.a (mon.0) 577 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:53.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[82331]: cluster 2024-04-13T07:44:52.329060+0000 mon.a (mon.0) 578 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:44:53.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[82331]: cluster 2024-04-13T07:44:52.346665+0000 mon.a (mon.0) 579 : cluster [INF] osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489] boot 2024-04-13T07:44:53.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[82331]: cluster 2024-04-13T07:44:52.346857+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-04-13T07:44:53.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[82331]: audit 2024-04-13T07:44:52.349605+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:44:53.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[82331]: cluster 2024-04-13T07:44:53.341187+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-04-13T07:44:53.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[84777]: cluster 2024-04-13T07:44:52.329005+0000 mon.a (mon.0) 577 : cluster [ERR] Health check update: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:44:53.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[84777]: cluster 2024-04-13T07:44:52.329060+0000 mon.a (mon.0) 578 : cluster [INF] Health check cleared: OSD_DOWN (was: 1 osds down) 2024-04-13T07:44:53.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[84777]: cluster 2024-04-13T07:44:52.346665+0000 mon.a (mon.0) 579 : cluster [INF] osd.4 [v2:172.21.15.203:6800/3455644489,v1:172.21.15.203:6801/3455644489] boot 2024-04-13T07:44:53.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[84777]: cluster 2024-04-13T07:44:52.346857+0000 mon.a (mon.0) 580 : cluster [DBG] osdmap e116: 8 total, 8 up, 8 in 2024-04-13T07:44:53.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[84777]: audit 2024-04-13T07:44:52.349605+0000 mon.a (mon.0) 581 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2024-04-13T07:44:53.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:53 smithi089 bash[84777]: cluster 2024-04-13T07:44:53.341187+0000 mon.a (mon.0) 582 : cluster [DBG] osdmap e117: 8 total, 8 up, 8 in 2024-04-13T07:44:54.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:54 smithi203 bash[68060]: cluster 2024-04-13T07:44:53.134626+0000 mgr.y (mgr.34107) 298 : cluster [DBG] pgmap v235: 73 pgs: 12 peering, 2 active+undersized+degraded, 19 active+undersized, 40 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:44:54.798 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:54 smithi089 bash[82331]: cluster 2024-04-13T07:44:53.134626+0000 mgr.y (mgr.34107) 298 : cluster [DBG] pgmap v235: 73 pgs: 12 peering, 2 active+undersized+degraded, 19 active+undersized, 40 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:44:54.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:54 smithi089 bash[84777]: cluster 2024-04-13T07:44:53.134626+0000 mgr.y (mgr.34107) 298 : cluster [DBG] pgmap v235: 73 pgs: 12 peering, 2 active+undersized+degraded, 19 active+undersized, 40 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:44:56.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:44:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:44:55] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:44:56.336 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:44:56.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:56 smithi203 bash[68060]: cluster 2024-04-13T07:44:55.135457+0000 mgr.y (mgr.34107) 299 : cluster [DBG] pgmap v237: 73 pgs: 12 peering, 2 active+undersized+degraded, 15 active+undersized, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:44:56.765 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:56 smithi203 bash[68060]: cluster 2024-04-13T07:44:55.640080+0000 mon.a (mon.0) 583 : cluster [WRN] Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:56.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:56 smithi089 bash[84777]: cluster 2024-04-13T07:44:55.135457+0000 mgr.y (mgr.34107) 299 : cluster [DBG] pgmap v237: 73 pgs: 12 peering, 2 active+undersized+degraded, 15 active+undersized, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:44:56.798 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:56 smithi089 bash[84777]: cluster 2024-04-13T07:44:55.640080+0000 mon.a (mon.0) 583 : cluster [WRN] Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:56.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:56 smithi089 bash[82331]: cluster 2024-04-13T07:44:55.135457+0000 mgr.y (mgr.34107) 299 : cluster [DBG] pgmap v237: 73 pgs: 12 peering, 2 active+undersized+degraded, 15 active+undersized, 44 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2024-04-13T07:44:56.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:56 smithi089 bash[82331]: cluster 2024-04-13T07:44:55.640080+0000 mon.a (mon.0) 583 : cluster [WRN] Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded (PG_DEGRADED) 2024-04-13T07:44:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:58 smithi203 bash[68060]: audit 2024-04-13T07:44:57.081001+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:58 smithi203 bash[68060]: audit 2024-04-13T07:44:57.084758+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:58.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:58 smithi203 bash[68060]: cluster 2024-04-13T07:44:57.136203+0000 mgr.y (mgr.34107) 300 : cluster [DBG] pgmap v238: 73 pgs: 12 peering, 61 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:58.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:58 smithi089 bash[84777]: audit 2024-04-13T07:44:57.081001+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:58.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:58 smithi089 bash[84777]: audit 2024-04-13T07:44:57.084758+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:58.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:58 smithi089 bash[84777]: cluster 2024-04-13T07:44:57.136203+0000 mgr.y (mgr.34107) 300 : cluster [DBG] pgmap v238: 73 pgs: 12 peering, 61 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:58.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:58 smithi089 bash[82331]: audit 2024-04-13T07:44:57.081001+0000 mon.a (mon.0) 584 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:44:58.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:58 smithi089 bash[82331]: audit 2024-04-13T07:44:57.084758+0000 mon.a (mon.0) 585 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:44:58.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:58 smithi089 bash[82331]: cluster 2024-04-13T07:44:57.136203+0000 mgr.y (mgr.34107) 300 : cluster [DBG] pgmap v238: 73 pgs: 12 peering, 61 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:44:59.289 INFO:teuthology.orchestra.run.smithi089.stdout:{ 2024-04-13T07:44:59.289 INFO:teuthology.orchestra.run.smithi089.stdout: "mon": { 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev)": 3 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: "mgr": { 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev)": 2 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: "osd": { 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1217-g7d36322c (7d36322c373c73b184fd252cd881b122d8c51a3b) reef (stable)": 3, 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev)": 5 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: "mds": { 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1217-g7d36322c (7d36322c373c73b184fd252cd881b122d8c51a3b) reef (stable)": 2 2024-04-13T07:44:59.290 INFO:teuthology.orchestra.run.smithi089.stdout: }, 2024-04-13T07:44:59.291 INFO:teuthology.orchestra.run.smithi089.stdout: "overall": { 2024-04-13T07:44:59.291 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 18.2.2-1217-g7d36322c (7d36322c373c73b184fd252cd881b122d8c51a3b) reef (stable)": 5, 2024-04-13T07:44:59.291 INFO:teuthology.orchestra.run.smithi089.stdout: "ceph version 19.0.0-2905-g5f22a497 (5f22a4974d6492df1d519913f5251731f043c582) squid (dev)": 10 2024-04-13T07:44:59.291 INFO:teuthology.orchestra.run.smithi089.stdout: } 2024-04-13T07:44:59.291 INFO:teuthology.orchestra.run.smithi089.stdout:} 2024-04-13T07:44:59.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:44:59 smithi089 bash[84777]: cluster 2024-04-13T07:44:58.076606+0000 mon.a (mon.0) 586 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded) 2024-04-13T07:44:59.298 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:44:59 smithi089 bash[82331]: cluster 2024-04-13T07:44:58.076606+0000 mon.a (mon.0) 586 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded) 2024-04-13T07:44:59.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:44:59 smithi203 bash[68060]: cluster 2024-04-13T07:44:58.076606+0000 mon.a (mon.0) 586 : cluster [INF] Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 5/84 objects degraded (5.952%), 2 pgs degraded) 2024-04-13T07:44:59.983 DEBUG:teuthology.parallel:result is None 2024-04-13T07:44:59.984 INFO:teuthology.run_tasks:Running task print... 2024-04-13T07:44:59.996 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2024-04-13T07:44:59.996 INFO:teuthology.run_tasks:Running task ceph.healthy... 2024-04-13T07:45:00.047 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2024-04-13T07:45:00.047 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2024-04-13T07:45:00.048 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph mgr dump --format=json 2024-04-13T07:45:00.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:00 smithi089 bash[82331]: cluster 2024-04-13T07:44:59.137055+0000 mgr.y (mgr.34107) 301 : cluster [DBG] pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:00.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:00 smithi089 bash[82331]: audit 2024-04-13T07:44:59.295428+0000 mon.a (mon.0) 587 : audit [DBG] from='client.? 172.21.15.89:0/1475106024' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:45:00.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:00 smithi089 bash[84777]: cluster 2024-04-13T07:44:59.137055+0000 mgr.y (mgr.34107) 301 : cluster [DBG] pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:00.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:00 smithi089 bash[84777]: audit 2024-04-13T07:44:59.295428+0000 mon.a (mon.0) 587 : audit [DBG] from='client.? 172.21.15.89:0/1475106024' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:45:00.353 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:00 smithi203 bash[68060]: cluster 2024-04-13T07:44:59.137055+0000 mgr.y (mgr.34107) 301 : cluster [DBG] pgmap v239: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:00.354 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:00 smithi203 bash[68060]: audit 2024-04-13T07:44:59.295428+0000 mon.a (mon.0) 587 : audit [DBG] from='client.? 172.21.15.89:0/1475106024' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2024-04-13T07:45:02.515 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[60252]: ts=2024-04-13T07:45:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:45:02.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: cluster 2024-04-13T07:45:01.137869+0000 mgr.y (mgr.34107) 302 : cluster [DBG] pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.164891+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.176121+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.178673+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.180639+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.190650+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.276159+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.278506+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.279699+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:45:02.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:02 smithi203 bash[68060]: audit 2024-04-13T07:45:01.287870+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: cluster 2024-04-13T07:45:01.137869+0000 mgr.y (mgr.34107) 302 : cluster [DBG] pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.164891+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.176121+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.178673+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.180639+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.190650+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.276159+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.278506+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.279699+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:45:02.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[82331]: audit 2024-04-13T07:45:01.287870+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: cluster 2024-04-13T07:45:01.137869+0000 mgr.y (mgr.34107) 302 : cluster [DBG] pgmap v240: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.164891+0000 mon.a (mon.0) 588 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.176121+0000 mon.a (mon.0) 589 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.178673+0000 mon.a (mon.0) 590 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.180639+0000 mon.a (mon.0) 591 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.190650+0000 mon.a (mon.0) 592 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.276159+0000 mon.a (mon.0) 593 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.278506+0000 mon.a (mon.0) 594 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.279699+0000 mon.a (mon.0) 595 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:45:02.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:02 smithi089 bash[84777]: audit 2024-04-13T07:45:01.287870+0000 mon.a (mon.0) 596 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:04.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:04 smithi203 bash[68060]: cluster 2024-04-13T07:45:03.138812+0000 mgr.y (mgr.34107) 303 : cluster [DBG] pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:04.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:04 smithi089 bash[84777]: cluster 2024-04-13T07:45:03.138812+0000 mgr.y (mgr.34107) 303 : cluster [DBG] pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:04.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:04 smithi089 bash[82331]: cluster 2024-04-13T07:45:03.138812+0000 mgr.y (mgr.34107) 303 : cluster [DBG] pgmap v241: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:04.789 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:05.970 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:45:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:45:05] "GET /metrics HTTP/1.1" 200 34078 "" "Prometheus/2.43.0" 2024-04-13T07:45:06.298 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:06 smithi089 bash[84777]: cluster 2024-04-13T07:45:05.139587+0000 mgr.y (mgr.34107) 304 : cluster [DBG] pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:06.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:06 smithi089 bash[82331]: cluster 2024-04-13T07:45:05.139587+0000 mgr.y (mgr.34107) 304 : cluster [DBG] pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:06.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:06 smithi203 bash[68060]: cluster 2024-04-13T07:45:05.139587+0000 mgr.y (mgr.34107) 304 : cluster [DBG] pgmap v242: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:07.609 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:45:08.281 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":38,"flags":0,"active_gid":34107,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6800","nonce":1865096045},{"type":"v1","addr":"172.21.15.89:6801","nonce":1865096045}]},"active_addr":"172.21.15.89:6801/1865096045","active_change":"2024-04-13T07:37:56.934399+0000","active_mgr_features":4540701547738038271,"available":true,"standbys":[{"gid":24769,"name":"x","mgr_features":4540701547738038271,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","read","upmap","upmap-read"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cephadm_log_destination":{"name":"cephadm_log_destination","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":["file","file,syslog","syslog"],"desc":"Destination for cephadm command's persistent logging","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/grafana:9.4.12","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:1.2.1","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_samba":{"name":"container_image_samba","type":"str","level":"advanced","flags":0,"default_value":"quay.io/samba.org/samba-server:devbuilds-centos-amd64","min":"","max":"","enum_allowed":[],"desc":"Samba/SMB container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"int","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"grafana_dashboards_path":{"name":"grafana_dashboards_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/grafana/dashboards/ceph-dashboard/","min":"","max":"","enum_allowed":[],"desc":"location of dashboards to include in grafana deployments","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"hw_monitoring":{"name":"hw_monitoring","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Deploy hw monitoring daemon on every host.","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"oob_default_addr":{"name":"oob_default_addr","type":"str","level":"advanced","flags":0,"default_value":"169.254.1.1","min":"","max":"","enum_allowed":[],"desc":"Default address for RedFish API (oob management).","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MANAGED_BY_CLUSTERS":{"name":"MANAGED_BY_CLUSTERS","type":"str","level":"advanced","flags":0,"default_value":"[]","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"MULTICLUSTER_CONFIG":{"name":"MULTICLUSTER_CONFIG","type":"str","level":"advanced","flags":0,"default_value":"{}","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"UNSAFE_TLS_v1_2":{"name":"UNSAFE_TLS_v1_2","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"periodic_async_work":{"name":"periodic_async_work","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Periodically check for async work","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_no_wait":{"name":"snapshot_clone_no_wait","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Reject subvolume clone request when cloner threads are busy","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.89:8443/","prometheus":"http://172.21.15.89:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"squid":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":92,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":3451413815}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":4292864937}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":2605666012}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.89:0","nonce":1375867528}]}]} 2024-04-13T07:45:08.286 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2024-04-13T07:45:08.287 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2024-04-13T07:45:08.287 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-13T07:45:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:08 smithi203 bash[68060]: cluster 2024-04-13T07:45:07.140414+0000 mgr.y (mgr.34107) 305 : cluster [DBG] pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:08 smithi203 bash[68060]: audit 2024-04-13T07:45:07.253978+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:08.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:08 smithi203 bash[68060]: audit 2024-04-13T07:45:07.609014+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/2501543743' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:45:08.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:08 smithi089 bash[84777]: cluster 2024-04-13T07:45:07.140414+0000 mgr.y (mgr.34107) 305 : cluster [DBG] pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:08.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:08 smithi089 bash[84777]: audit 2024-04-13T07:45:07.253978+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:08 smithi089 bash[84777]: audit 2024-04-13T07:45:07.609014+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/2501543743' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:45:08.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:08 smithi089 bash[82331]: cluster 2024-04-13T07:45:07.140414+0000 mgr.y (mgr.34107) 305 : cluster [DBG] pgmap v243: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:08.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:08 smithi089 bash[82331]: audit 2024-04-13T07:45:07.253978+0000 mon.a (mon.0) 597 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:08.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:08 smithi089 bash[82331]: audit 2024-04-13T07:45:07.609014+0000 mon.c (mon.2) 22 : audit [DBG] from='client.? 172.21.15.89:0/2501543743' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2024-04-13T07:45:10.515 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:10 smithi203 bash[68060]: cluster 2024-04-13T07:45:09.141319+0000 mgr.y (mgr.34107) 306 : cluster [DBG] pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:10.548 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:10 smithi089 bash[82331]: cluster 2024-04-13T07:45:09.141319+0000 mgr.y (mgr.34107) 306 : cluster [DBG] pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:10.548 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:10 smithi089 bash[84777]: cluster 2024-04-13T07:45:09.141319+0000 mgr.y (mgr.34107) 306 : cluster [DBG] pgmap v244: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:12.649 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:45:12 smithi203 bash[60252]: ts=2024-04-13T07:45:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:45:13.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:12 smithi203 bash[68060]: cluster 2024-04-13T07:45:11.142132+0000 mgr.y (mgr.34107) 307 : cluster [DBG] pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:13.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:12 smithi203 bash[68060]: audit 2024-04-13T07:45:12.080928+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:13.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:12 smithi203 bash[68060]: audit 2024-04-13T07:45:12.083160+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:13.030 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:13.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:12 smithi089 bash[84777]: cluster 2024-04-13T07:45:11.142132+0000 mgr.y (mgr.34107) 307 : cluster [DBG] pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:13.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:12 smithi089 bash[84777]: audit 2024-04-13T07:45:12.080928+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:13.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:12 smithi089 bash[84777]: audit 2024-04-13T07:45:12.083160+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:13.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:12 smithi089 bash[82331]: cluster 2024-04-13T07:45:11.142132+0000 mgr.y (mgr.34107) 307 : cluster [DBG] pgmap v245: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:13.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:12 smithi089 bash[82331]: audit 2024-04-13T07:45:12.080928+0000 mon.a (mon.0) 598 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:45:13.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:12 smithi089 bash[82331]: audit 2024-04-13T07:45:12.083160+0000 mon.a (mon.0) 599 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:15.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:14 smithi203 bash[68060]: cluster 2024-04-13T07:45:13.143163+0000 mgr.y (mgr.34107) 308 : cluster [DBG] pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:15.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:14 smithi089 bash[82331]: cluster 2024-04-13T07:45:13.143163+0000 mgr.y (mgr.34107) 308 : cluster [DBG] pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:15.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:14 smithi089 bash[84777]: cluster 2024-04-13T07:45:13.143163+0000 mgr.y (mgr.34107) 308 : cluster [DBG] pgmap v246: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:15.821 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:45:15.821 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":117,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","created":"2024-04-13T07:19:16.936150+0000","modified":"2024-04-13T07:44:53.331354+0000","last_up_change":"2024-04-13T07:44:52.328034+0000","last_in_change":"2024-04-13T07:28:13.615937+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-13T07:24:24.430140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-13T07:30:20.226256+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":87,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-13T07:32:01.556724+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-13T07:32:02.440617+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9de691f5-25ed-472d-b6dd-dc3908fa85c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":92,"up_from":96,"up_thru":116,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":3498175744},{"type":"v1","addr":"172.21.15.89:6803","nonce":3498175744}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":3498175744},{"type":"v1","addr":"172.21.15.89:6805","nonce":3498175744}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":3498175744},{"type":"v1","addr":"172.21.15.89:6809","nonce":3498175744}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":3498175744},{"type":"v1","addr":"172.21.15.89:6807","nonce":3498175744}]},"public_addr":"172.21.15.89:6803/3498175744","cluster_addr":"172.21.15.89:6805/3498175744","heartbeat_back_addr":"172.21.15.89:6809/3498175744","heartbeat_front_addr":"172.21.15.89:6807/3498175744","state":["exists","up"]},{"osd":1,"uuid":"ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":97,"up_from":101,"up_thru":116,"down_at":98,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2607331631},{"type":"v1","addr":"172.21.15.89:6811","nonce":2607331631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2607331631},{"type":"v1","addr":"172.21.15.89:6813","nonce":2607331631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2607331631},{"type":"v1","addr":"172.21.15.89:6817","nonce":2607331631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2607331631},{"type":"v1","addr":"172.21.15.89:6815","nonce":2607331631}]},"public_addr":"172.21.15.89:6811/2607331631","cluster_addr":"172.21.15.89:6813/2607331631","heartbeat_back_addr":"172.21.15.89:6817/2607331631","heartbeat_front_addr":"172.21.15.89:6815/2607331631","state":["exists","up"]},{"osd":2,"uuid":"f86d6aea-97d3-4d75-99d0-6e5781d7a990","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":102,"up_from":106,"up_thru":116,"down_at":103,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":2720399818},{"type":"v1","addr":"172.21.15.89:6819","nonce":2720399818}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":2720399818},{"type":"v1","addr":"172.21.15.89:6821","nonce":2720399818}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":2720399818},{"type":"v1","addr":"172.21.15.89:6825","nonce":2720399818}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":2720399818},{"type":"v1","addr":"172.21.15.89:6823","nonce":2720399818}]},"public_addr":"172.21.15.89:6819/2720399818","cluster_addr":"172.21.15.89:6821/2720399818","heartbeat_back_addr":"172.21.15.89:6825/2720399818","heartbeat_front_addr":"172.21.15.89:6823/2720399818","state":["exists","up"]},{"osd":3,"uuid":"37c263d3-d642-4cfa-99da-5a33e1bef16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":107,"up_from":111,"up_thru":116,"down_at":108,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":367454897},{"type":"v1","addr":"172.21.15.89:6827","nonce":367454897}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":367454897},{"type":"v1","addr":"172.21.15.89:6829","nonce":367454897}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":367454897},{"type":"v1","addr":"172.21.15.89:6833","nonce":367454897}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":367454897},{"type":"v1","addr":"172.21.15.89:6831","nonce":367454897}]},"public_addr":"172.21.15.89:6827/367454897","cluster_addr":"172.21.15.89:6829/367454897","heartbeat_back_addr":"172.21.15.89:6833/367454897","heartbeat_front_addr":"172.21.15.89:6831/367454897","state":["exists","up"]},{"osd":4,"uuid":"c1befaa2-c9f5-4023-9909-0577509ce44d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":112,"up_from":116,"up_thru":116,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":3455644489},{"type":"v1","addr":"172.21.15.203:6801","nonce":3455644489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":3455644489},{"type":"v1","addr":"172.21.15.203:6803","nonce":3455644489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":3455644489},{"type":"v1","addr":"172.21.15.203:6807","nonce":3455644489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":3455644489},{"type":"v1","addr":"172.21.15.203:6805","nonce":3455644489}]},"public_addr":"172.21.15.203:6801/3455644489","cluster_addr":"172.21.15.203:6803/3455644489","heartbeat_back_addr":"172.21.15.203:6807/3455644489","heartbeat_front_addr":"172.21.15.203:6805/3455644489","state":["exists","up"]},{"osd":5,"uuid":"b45987d4-fc0f-45ad-9f0d-49240d898bfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6809","nonce":209623809}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6811","nonce":209623809}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6815","nonce":209623809}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6813","nonce":209623809}]},"public_addr":"172.21.15.203:6809/209623809","cluster_addr":"172.21.15.203:6811/209623809","heartbeat_back_addr":"172.21.15.203:6815/209623809","heartbeat_front_addr":"172.21.15.203:6813/209623809","state":["exists","up"]},{"osd":6,"uuid":"91bbf36b-799c-49df-8fbf-42a958699642","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6817","nonce":2138665907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6819","nonce":2138665907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6823","nonce":2138665907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6821","nonce":2138665907}]},"public_addr":"172.21.15.203:6817/2138665907","cluster_addr":"172.21.15.203:6819/2138665907","heartbeat_back_addr":"172.21.15.203:6823/2138665907","heartbeat_front_addr":"172.21.15.203:6821/2138665907","state":["exists","up"]},{"osd":7,"uuid":"aa9fb5c4-06e4-45ca-85f8-f2a96be15086","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6825","nonce":3848298771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6827","nonce":3848298771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6831","nonce":3848298771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6829","nonce":3848298771}]},"public_addr":"172.21.15.203:6825/3848298771","cluster_addr":"172.21.15.203:6827/3848298771","heartbeat_back_addr":"172.21.15.203:6831/3848298771","heartbeat_front_addr":"172.21.15.203:6829/3848298771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:22:41.252314+0000","dead_epoch":92},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:23:31.382195+0000","dead_epoch":97},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:24:20.791478+0000","dead_epoch":102},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:10.117632+0000","dead_epoch":107},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:59.130696+0000","dead_epoch":112},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:26:51.940707+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:27:44.966139+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:28:38.869265+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/4212136951":"2024-04-14T07:37:56.933840+0000","172.21.15.203:0/275131096":"2024-04-14T07:36:52.177539+0000","172.21.15.89:0/4124851388":"2024-04-14T07:35:58.581731+0000","172.21.15.203:0/1013314899":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6835/611307576":"2024-04-14T07:32:10.835116+0000","172.21.15.89:0/808783885":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6834/611307576":"2024-04-14T07:32:10.835116+0000","172.21.15.89:6801/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/3115322972":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/716328009":"2024-04-14T07:37:56.933840+0000","172.21.15.203:0/1665565419":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6801/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/1853920825":"2024-04-14T07:35:58.581731+0000","172.21.15.89:0/2936084513":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1139384373":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1817808921":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/28202064":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/2670200407":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/2680580613":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/1215893273":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/2897907290":"2024-04-14T07:35:58.581731+0000","172.21.15.89:6801/2897907290":"2024-04-14T07:35:58.581731+0000","172.21.15.203:0/1055122310":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6800/3021779290":"2024-04-14T07:37:56.933840+0000","172.21.15.203:0/1613796417":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6801/3021779290":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/4260385376":"2024-04-14T07:35:58.581731+0000","172.21.15.89:0/2329788515":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/3574204772":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1725920612":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/3991952503":"2024-04-14T07:35:58.581731+0000","172.21.15.203:0/968406663":"2024-04-14T07:36:52.177539+0000","172.21.15.89:0/868096906":"2024-04-14T07:19:48.376853+0000","172.21.15.203:6834/711882650":"2024-04-14T07:36:52.177539+0000","172.21.15.203:6835/711882650":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6800/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6801/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2483652022":"2024-04-14T07:29:38.824647+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-13T07:45:16.298 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:45:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:45:15] "GET /metrics HTTP/1.1" 200 34076 "" "Prometheus/2.43.0" 2024-04-13T07:45:16.525 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2024-04-13T07:45:16.526 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd dump --format=json 2024-04-13T07:45:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:16 smithi089 bash[84777]: cluster 2024-04-13T07:45:15.144003+0000 mgr.y (mgr.34107) 309 : cluster [DBG] pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:16 smithi089 bash[84777]: audit 2024-04-13T07:45:15.825107+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/3262843101' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:45:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:16 smithi089 bash[82331]: cluster 2024-04-13T07:45:15.144003+0000 mgr.y (mgr.34107) 309 : cluster [DBG] pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:16 smithi089 bash[82331]: audit 2024-04-13T07:45:15.825107+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/3262843101' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:45:17.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:16 smithi203 bash[68060]: cluster 2024-04-13T07:45:15.144003+0000 mgr.y (mgr.34107) 309 : cluster [DBG] pgmap v247: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:17.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:16 smithi203 bash[68060]: audit 2024-04-13T07:45:15.825107+0000 mon.c (mon.2) 23 : audit [DBG] from='client.? 172.21.15.89:0/3262843101' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:45:19.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:18 smithi203 bash[68060]: cluster 2024-04-13T07:45:17.144815+0000 mgr.y (mgr.34107) 310 : cluster [DBG] pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:19.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:18 smithi089 bash[84777]: cluster 2024-04-13T07:45:17.144815+0000 mgr.y (mgr.34107) 310 : cluster [DBG] pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:19.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:18 smithi089 bash[82331]: cluster 2024-04-13T07:45:17.144815+0000 mgr.y (mgr.34107) 310 : cluster [DBG] pgmap v248: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:21.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:20 smithi203 bash[68060]: cluster 2024-04-13T07:45:19.145738+0000 mgr.y (mgr.34107) 311 : cluster [DBG] pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:21.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:20 smithi089 bash[84777]: cluster 2024-04-13T07:45:19.145738+0000 mgr.y (mgr.34107) 311 : cluster [DBG] pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:21.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:20 smithi089 bash[82331]: cluster 2024-04-13T07:45:19.145738+0000 mgr.y (mgr.34107) 311 : cluster [DBG] pgmap v249: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:21.255 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:22.669 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:45:22 smithi203 bash[60252]: ts=2024-04-13T07:45:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:45:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:22 smithi089 bash[82331]: cluster 2024-04-13T07:45:21.146562+0000 mgr.y (mgr.34107) 312 : cluster [DBG] pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:22 smithi089 bash[84777]: cluster 2024-04-13T07:45:21.146562+0000 mgr.y (mgr.34107) 312 : cluster [DBG] pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:23.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:22 smithi203 bash[68060]: cluster 2024-04-13T07:45:21.146562+0000 mgr.y (mgr.34107) 312 : cluster [DBG] pgmap v250: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:24.158 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:45:24.159 INFO:teuthology.orchestra.run.smithi089.stdout:{"epoch":117,"fsid":"ec50a262-f965-11ee-bc8d-c7b262605968","created":"2024-04-13T07:19:16.936150+0000","modified":"2024-04-13T07:44:53.331354+0000","last_up_change":"2024-04-13T07:44:52.328034+0000","last_in_change":"2024-04-13T07:28:13.615937+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":23,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"reef","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2024-04-13T07:24:24.430140+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_type":"Fair distribution","score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2024-04-13T07:30:20.226256+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"87","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":87,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_type":"Fair distribution","score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2024-04-13T07:32:01.556724+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"68","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2024-04-13T07:32:02.440617+0000","flags":131073,"flags_names":"hashpspool,bulk","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"69","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_type":"Fair distribution","score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"9de691f5-25ed-472d-b6dd-dc3908fa85c0","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":92,"up_from":96,"up_thru":116,"down_at":93,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6802","nonce":3498175744},{"type":"v1","addr":"172.21.15.89:6803","nonce":3498175744}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6804","nonce":3498175744},{"type":"v1","addr":"172.21.15.89:6805","nonce":3498175744}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6808","nonce":3498175744},{"type":"v1","addr":"172.21.15.89:6809","nonce":3498175744}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6806","nonce":3498175744},{"type":"v1","addr":"172.21.15.89:6807","nonce":3498175744}]},"public_addr":"172.21.15.89:6803/3498175744","cluster_addr":"172.21.15.89:6805/3498175744","heartbeat_back_addr":"172.21.15.89:6809/3498175744","heartbeat_front_addr":"172.21.15.89:6807/3498175744","state":["exists","up"]},{"osd":1,"uuid":"ab7d4a76-467f-48ea-b7cf-e38bdc2b39b2","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":97,"up_from":101,"up_thru":116,"down_at":98,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6810","nonce":2607331631},{"type":"v1","addr":"172.21.15.89:6811","nonce":2607331631}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6812","nonce":2607331631},{"type":"v1","addr":"172.21.15.89:6813","nonce":2607331631}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6816","nonce":2607331631},{"type":"v1","addr":"172.21.15.89:6817","nonce":2607331631}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6814","nonce":2607331631},{"type":"v1","addr":"172.21.15.89:6815","nonce":2607331631}]},"public_addr":"172.21.15.89:6811/2607331631","cluster_addr":"172.21.15.89:6813/2607331631","heartbeat_back_addr":"172.21.15.89:6817/2607331631","heartbeat_front_addr":"172.21.15.89:6815/2607331631","state":["exists","up"]},{"osd":2,"uuid":"f86d6aea-97d3-4d75-99d0-6e5781d7a990","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":102,"up_from":106,"up_thru":116,"down_at":103,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6818","nonce":2720399818},{"type":"v1","addr":"172.21.15.89:6819","nonce":2720399818}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6820","nonce":2720399818},{"type":"v1","addr":"172.21.15.89:6821","nonce":2720399818}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6824","nonce":2720399818},{"type":"v1","addr":"172.21.15.89:6825","nonce":2720399818}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6822","nonce":2720399818},{"type":"v1","addr":"172.21.15.89:6823","nonce":2720399818}]},"public_addr":"172.21.15.89:6819/2720399818","cluster_addr":"172.21.15.89:6821/2720399818","heartbeat_back_addr":"172.21.15.89:6825/2720399818","heartbeat_front_addr":"172.21.15.89:6823/2720399818","state":["exists","up"]},{"osd":3,"uuid":"37c263d3-d642-4cfa-99da-5a33e1bef16e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":26,"last_clean_end":107,"up_from":111,"up_thru":116,"down_at":108,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6826","nonce":367454897},{"type":"v1","addr":"172.21.15.89:6827","nonce":367454897}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6828","nonce":367454897},{"type":"v1","addr":"172.21.15.89:6829","nonce":367454897}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6832","nonce":367454897},{"type":"v1","addr":"172.21.15.89:6833","nonce":367454897}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.89:6830","nonce":367454897},{"type":"v1","addr":"172.21.15.89:6831","nonce":367454897}]},"public_addr":"172.21.15.89:6827/367454897","cluster_addr":"172.21.15.89:6829/367454897","heartbeat_back_addr":"172.21.15.89:6833/367454897","heartbeat_front_addr":"172.21.15.89:6831/367454897","state":["exists","up"]},{"osd":4,"uuid":"c1befaa2-c9f5-4023-9909-0577509ce44d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":31,"last_clean_end":112,"up_from":116,"up_thru":116,"down_at":113,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6800","nonce":3455644489},{"type":"v1","addr":"172.21.15.203:6801","nonce":3455644489}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6802","nonce":3455644489},{"type":"v1","addr":"172.21.15.203:6803","nonce":3455644489}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6806","nonce":3455644489},{"type":"v1","addr":"172.21.15.203:6807","nonce":3455644489}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6804","nonce":3455644489},{"type":"v1","addr":"172.21.15.203:6805","nonce":3455644489}]},"public_addr":"172.21.15.203:6801/3455644489","cluster_addr":"172.21.15.203:6803/3455644489","heartbeat_back_addr":"172.21.15.203:6807/3455644489","heartbeat_front_addr":"172.21.15.203:6805/3455644489","state":["exists","up"]},{"osd":5,"uuid":"b45987d4-fc0f-45ad-9f0d-49240d898bfc","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":39,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6808","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6809","nonce":209623809}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6810","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6811","nonce":209623809}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6814","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6815","nonce":209623809}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6812","nonce":209623809},{"type":"v1","addr":"172.21.15.203:6813","nonce":209623809}]},"public_addr":"172.21.15.203:6809/209623809","cluster_addr":"172.21.15.203:6811/209623809","heartbeat_back_addr":"172.21.15.203:6815/209623809","heartbeat_front_addr":"172.21.15.203:6813/209623809","state":["exists","up"]},{"osd":6,"uuid":"91bbf36b-799c-49df-8fbf-42a958699642","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":47,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6816","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6817","nonce":2138665907}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6818","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6819","nonce":2138665907}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6822","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6823","nonce":2138665907}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6820","nonce":2138665907},{"type":"v1","addr":"172.21.15.203:6821","nonce":2138665907}]},"public_addr":"172.21.15.203:6817/2138665907","cluster_addr":"172.21.15.203:6819/2138665907","heartbeat_back_addr":"172.21.15.203:6823/2138665907","heartbeat_front_addr":"172.21.15.203:6821/2138665907","state":["exists","up"]},{"osd":7,"uuid":"aa9fb5c4-06e4-45ca-85f8-f2a96be15086","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":55,"up_thru":116,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6824","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6825","nonce":3848298771}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6826","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6827","nonce":3848298771}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6830","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6831","nonce":3848298771}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.203:6828","nonce":3848298771},{"type":"v1","addr":"172.21.15.203:6829","nonce":3848298771}]},"public_addr":"172.21.15.203:6825/3848298771","cluster_addr":"172.21.15.203:6827/3848298771","heartbeat_back_addr":"172.21.15.203:6831/3848298771","heartbeat_front_addr":"172.21.15.203:6829/3848298771","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:22:41.252314+0000","dead_epoch":92},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:23:31.382195+0000","dead_epoch":97},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:24:20.791478+0000","dead_epoch":102},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:10.117632+0000","dead_epoch":107},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540701547738038271,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:25:59.130696+0000","dead_epoch":112},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:26:51.940707+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:27:44.966139+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2024-04-13T07:28:38.869265+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.f","mappings":[{"from":1,"to":2}]},{"pgid":"4.17","mappings":[{"from":1,"to":5}]},{"pgid":"4.1d","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.89:0/4212136951":"2024-04-14T07:37:56.933840+0000","172.21.15.203:0/275131096":"2024-04-14T07:36:52.177539+0000","172.21.15.89:0/4124851388":"2024-04-14T07:35:58.581731+0000","172.21.15.203:0/1013314899":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6835/611307576":"2024-04-14T07:32:10.835116+0000","172.21.15.89:0/808783885":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6834/611307576":"2024-04-14T07:32:10.835116+0000","172.21.15.89:6801/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/3115322972":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/716328009":"2024-04-14T07:37:56.933840+0000","172.21.15.203:0/1665565419":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6801/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/1853920825":"2024-04-14T07:35:58.581731+0000","172.21.15.89:0/2936084513":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/1711773703":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1139384373":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1817808921":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/28202064":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/2670200407":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/2680580613":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6800/1857363809":"2024-04-14T07:20:19.796707+0000","172.21.15.89:0/1215893273":"2024-04-14T07:20:19.796707+0000","172.21.15.89:6800/2897907290":"2024-04-14T07:35:58.581731+0000","172.21.15.89:6801/2897907290":"2024-04-14T07:35:58.581731+0000","172.21.15.203:0/1055122310":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6800/3021779290":"2024-04-14T07:37:56.933840+0000","172.21.15.203:0/1613796417":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6801/3021779290":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/4260385376":"2024-04-14T07:35:58.581731+0000","172.21.15.89:0/2329788515":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/3574204772":"2024-04-14T07:29:38.824647+0000","172.21.15.89:0/1725920612":"2024-04-14T07:37:56.933840+0000","172.21.15.89:0/3991952503":"2024-04-14T07:35:58.581731+0000","172.21.15.203:0/968406663":"2024-04-14T07:36:52.177539+0000","172.21.15.89:0/868096906":"2024-04-14T07:19:48.376853+0000","172.21.15.203:6834/711882650":"2024-04-14T07:36:52.177539+0000","172.21.15.203:6835/711882650":"2024-04-14T07:36:52.177539+0000","172.21.15.89:6800/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:6801/1964774827":"2024-04-14T07:19:48.376853+0000","172.21.15.89:0/2483652022":"2024-04-14T07:29:38.824647+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2024-04-13T07:45:24.864 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.0 flush_pg_stats 2024-04-13T07:45:24.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.1 flush_pg_stats 2024-04-13T07:45:24.865 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.2 flush_pg_stats 2024-04-13T07:45:24.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.3 flush_pg_stats 2024-04-13T07:45:24.866 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.4 flush_pg_stats 2024-04-13T07:45:24.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.5 flush_pg_stats 2024-04-13T07:45:24.867 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.6 flush_pg_stats 2024-04-13T07:45:24.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph tell osd.7 flush_pg_stats 2024-04-13T07:45:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:24 smithi203 bash[68060]: cluster 2024-04-13T07:45:23.147558+0000 mgr.y (mgr.34107) 313 : cluster [DBG] pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:25.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:24 smithi203 bash[68060]: audit 2024-04-13T07:45:24.163164+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/2486196282' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:45:25.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:24 smithi089 bash[82331]: cluster 2024-04-13T07:45:23.147558+0000 mgr.y (mgr.34107) 313 : cluster [DBG] pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:25.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:24 smithi089 bash[82331]: audit 2024-04-13T07:45:24.163164+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/2486196282' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:45:25.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:24 smithi089 bash[84777]: cluster 2024-04-13T07:45:23.147558+0000 mgr.y (mgr.34107) 313 : cluster [DBG] pgmap v251: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:25.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:24 smithi089 bash[84777]: audit 2024-04-13T07:45:24.163164+0000 mon.c (mon.2) 24 : audit [DBG] from='client.? 172.21.15.89:0/2486196282' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2024-04-13T07:45:26.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:45:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:45:25] "GET /metrics HTTP/1.1" 200 34076 "" "Prometheus/2.43.0" 2024-04-13T07:45:27.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:26 smithi203 bash[68060]: cluster 2024-04-13T07:45:25.148072+0000 mgr.y (mgr.34107) 314 : cluster [DBG] pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:27.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:26 smithi089 bash[82331]: cluster 2024-04-13T07:45:25.148072+0000 mgr.y (mgr.34107) 314 : cluster [DBG] pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:27.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:26 smithi089 bash[84777]: cluster 2024-04-13T07:45:25.148072+0000 mgr.y (mgr.34107) 314 : cluster [DBG] pgmap v252: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:28.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:27 smithi203 bash[68060]: audit 2024-04-13T07:45:27.075264+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:28.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:27 smithi089 bash[82331]: audit 2024-04-13T07:45:27.075264+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:28.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:27 smithi089 bash[84777]: audit 2024-04-13T07:45:27.075264+0000 mon.a (mon.0) 600 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:29.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:28 smithi203 bash[68060]: cluster 2024-04-13T07:45:27.148843+0000 mgr.y (mgr.34107) 315 : cluster [DBG] pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:29.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:28 smithi089 bash[82331]: cluster 2024-04-13T07:45:27.148843+0000 mgr.y (mgr.34107) 315 : cluster [DBG] pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:29.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:28 smithi089 bash[84777]: cluster 2024-04-13T07:45:27.148843+0000 mgr.y (mgr.34107) 315 : cluster [DBG] pgmap v253: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:29.688 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:29.689 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:29.689 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:29.689 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:29.690 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:29.690 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:29.693 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:29.695 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:30.951 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:30 smithi089 bash[84777]: cluster 2024-04-13T07:45:29.149783+0000 mgr.y (mgr.34107) 316 : cluster [DBG] pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:30.951 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:30 smithi089 bash[82331]: cluster 2024-04-13T07:45:29.149783+0000 mgr.y (mgr.34107) 316 : cluster [DBG] pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:31.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:30 smithi203 bash[68060]: cluster 2024-04-13T07:45:29.149783+0000 mgr.y (mgr.34107) 316 : cluster [DBG] pgmap v254: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:32.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:45:32 smithi203 bash[60252]: ts=2024-04-13T07:45:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:45:32.964 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:32 smithi089 bash[84777]: cluster 2024-04-13T07:45:31.150381+0000 mgr.y (mgr.34107) 317 : cluster [DBG] pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:32.964 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:32 smithi089 bash[82331]: cluster 2024-04-13T07:45:31.150381+0000 mgr.y (mgr.34107) 317 : cluster [DBG] pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:33.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:32 smithi203 bash[68060]: cluster 2024-04-13T07:45:31.150381+0000 mgr.y (mgr.34107) 317 : cluster [DBG] pgmap v255: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:34.964 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:34 smithi089 bash[82331]: cluster 2024-04-13T07:45:33.151278+0000 mgr.y (mgr.34107) 318 : cluster [DBG] pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:34.965 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:34 smithi089 bash[84777]: cluster 2024-04-13T07:45:33.151278+0000 mgr.y (mgr.34107) 318 : cluster [DBG] pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:35.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:34 smithi203 bash[68060]: cluster 2024-04-13T07:45:33.151278+0000 mgr.y (mgr.34107) 318 : cluster [DBG] pgmap v256: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:35.982 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:45:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:45:35] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.43.0" 2024-04-13T07:45:37.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:36 smithi203 bash[68060]: cluster 2024-04-13T07:45:35.151944+0000 mgr.y (mgr.34107) 319 : cluster [DBG] pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:37.048 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:36 smithi089 bash[82331]: cluster 2024-04-13T07:45:35.151944+0000 mgr.y (mgr.34107) 319 : cluster [DBG] pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:37.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:36 smithi089 bash[84777]: cluster 2024-04-13T07:45:35.151944+0000 mgr.y (mgr.34107) 319 : cluster [DBG] pgmap v257: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:39.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:38 smithi203 bash[68060]: cluster 2024-04-13T07:45:37.152737+0000 mgr.y (mgr.34107) 320 : cluster [DBG] pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:39.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:38 smithi089 bash[82331]: cluster 2024-04-13T07:45:37.152737+0000 mgr.y (mgr.34107) 320 : cluster [DBG] pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:39.032 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:38 smithi089 bash[84777]: cluster 2024-04-13T07:45:37.152737+0000 mgr.y (mgr.34107) 320 : cluster [DBG] pgmap v258: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:41.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:40 smithi203 bash[68060]: cluster 2024-04-13T07:45:39.153285+0000 mgr.y (mgr.34107) 321 : cluster [DBG] pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:41.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:40 smithi089 bash[84777]: cluster 2024-04-13T07:45:39.153285+0000 mgr.y (mgr.34107) 321 : cluster [DBG] pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:41.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:40 smithi089 bash[82331]: cluster 2024-04-13T07:45:39.153285+0000 mgr.y (mgr.34107) 321 : cluster [DBG] pgmap v259: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:42.380 INFO:teuthology.orchestra.run.smithi089.stdout:455266533398 2024-04-13T07:45:42.381 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.2 2024-04-13T07:45:42.461 INFO:teuthology.orchestra.run.smithi089.stdout:433791696924 2024-04-13T07:45:42.462 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.1 2024-04-13T07:45:42.716 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:45:42 smithi203 bash[60252]: ts=2024-04-13T07:45:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:45:42.982 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:42 smithi089 bash[84777]: cluster 2024-04-13T07:45:41.154166+0000 mgr.y (mgr.34107) 322 : cluster [DBG] pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:42.983 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:42 smithi089 bash[84777]: audit 2024-04-13T07:45:42.075796+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:42.983 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:42 smithi089 bash[82331]: cluster 2024-04-13T07:45:41.154166+0000 mgr.y (mgr.34107) 322 : cluster [DBG] pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:42.983 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:42 smithi089 bash[82331]: audit 2024-04-13T07:45:42.075796+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:43.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:42 smithi203 bash[68060]: cluster 2024-04-13T07:45:41.154166+0000 mgr.y (mgr.34107) 322 : cluster [DBG] pgmap v260: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:43.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:42 smithi203 bash[68060]: audit 2024-04-13T07:45:42.075796+0000 mon.a (mon.0) 601 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:43.783 INFO:teuthology.orchestra.run.smithi089.stdout:498216206347 2024-04-13T07:45:43.783 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.4 2024-04-13T07:45:43.934 INFO:teuthology.orchestra.run.smithi089.stdout:201863463129 2024-04-13T07:45:43.934 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.6 2024-04-13T07:45:44.999 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:44 smithi089 bash[84777]: cluster 2024-04-13T07:45:43.155163+0000 mgr.y (mgr.34107) 323 : cluster [DBG] pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:44.999 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:44 smithi089 bash[82331]: cluster 2024-04-13T07:45:43.155163+0000 mgr.y (mgr.34107) 323 : cluster [DBG] pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:45.015 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:44 smithi203 bash[68060]: cluster 2024-04-13T07:45:43.155163+0000 mgr.y (mgr.34107) 323 : cluster [DBG] pgmap v261: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:45.121 INFO:teuthology.orchestra.run.smithi089.stdout:236223201487 2024-04-13T07:45:45.121 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.7 2024-04-13T07:45:46.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:45:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:45:45] "GET /metrics HTTP/1.1" 200 34076 "" "Prometheus/2.43.0" 2024-04-13T07:45:46.429 INFO:teuthology.orchestra.run.smithi089.stdout:167503724773 2024-04-13T07:45:46.429 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.5 2024-04-13T07:45:46.509 INFO:teuthology.orchestra.run.smithi089.stdout:476741369874 2024-04-13T07:45:46.509 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.3 2024-04-13T07:45:46.600 INFO:teuthology.orchestra.run.smithi089.stdout:412316860465 2024-04-13T07:45:46.601 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph osd last-stat-seq osd.0 2024-04-13T07:45:47.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:46 smithi203 bash[68060]: cluster 2024-04-13T07:45:45.155672+0000 mgr.y (mgr.34107) 324 : cluster [DBG] pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:47.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:46 smithi089 bash[84777]: cluster 2024-04-13T07:45:45.155672+0000 mgr.y (mgr.34107) 324 : cluster [DBG] pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:47.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:46 smithi089 bash[82331]: cluster 2024-04-13T07:45:45.155672+0000 mgr.y (mgr.34107) 324 : cluster [DBG] pgmap v262: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:49.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:48 smithi203 bash[68060]: cluster 2024-04-13T07:45:47.156292+0000 mgr.y (mgr.34107) 325 : cluster [DBG] pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:49.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:48 smithi089 bash[84777]: cluster 2024-04-13T07:45:47.156292+0000 mgr.y (mgr.34107) 325 : cluster [DBG] pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:49.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:48 smithi089 bash[82331]: cluster 2024-04-13T07:45:47.156292+0000 mgr.y (mgr.34107) 325 : cluster [DBG] pgmap v263: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:49.127 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:49.128 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:49.130 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:49.130 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:50.147 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:51.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:50 smithi203 bash[68060]: cluster 2024-04-13T07:45:49.159080+0000 mgr.y (mgr.34107) 326 : cluster [DBG] pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:51.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:50 smithi089 bash[82331]: cluster 2024-04-13T07:45:49.159080+0000 mgr.y (mgr.34107) 326 : cluster [DBG] pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:50 smithi089 bash[84777]: cluster 2024-04-13T07:45:49.159080+0000 mgr.y (mgr.34107) 326 : cluster [DBG] pgmap v264: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:51.173 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:51.173 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:51.174 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:45:52.735 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:45:52 smithi203 bash[60252]: ts=2024-04-13T07:45:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:45:52.990 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:52 smithi089 bash[82331]: cluster 2024-04-13T07:45:51.159597+0000 mgr.y (mgr.34107) 327 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:52.990 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:52 smithi089 bash[84777]: cluster 2024-04-13T07:45:51.159597+0000 mgr.y (mgr.34107) 327 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:53.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:52 smithi203 bash[68060]: cluster 2024-04-13T07:45:51.159597+0000 mgr.y (mgr.34107) 327 : cluster [DBG] pgmap v265: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:55.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:54 smithi203 bash[68060]: cluster 2024-04-13T07:45:53.160190+0000 mgr.y (mgr.34107) 328 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:55.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:54 smithi089 bash[84777]: cluster 2024-04-13T07:45:53.160190+0000 mgr.y (mgr.34107) 328 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:55.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:54 smithi089 bash[82331]: cluster 2024-04-13T07:45:53.160190+0000 mgr.y (mgr.34107) 328 : cluster [DBG] pgmap v266: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:55.516 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:45:55 smithi203 bash[49777]: logger=sqlstore.transactions t=2024-04-13T07:45:55.175885523Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2024-04-13T07:45:56.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:45:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:45:55] "GET /metrics HTTP/1.1" 200 34076 "" "Prometheus/2.43.0" 2024-04-13T07:45:56.995 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:56 smithi089 bash[84777]: cluster 2024-04-13T07:45:55.160832+0000 mgr.y (mgr.34107) 329 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:56.995 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:56 smithi089 bash[82331]: cluster 2024-04-13T07:45:55.160832+0000 mgr.y (mgr.34107) 329 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:57.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:56 smithi203 bash[68060]: cluster 2024-04-13T07:45:55.160832+0000 mgr.y (mgr.34107) 329 : cluster [DBG] pgmap v267: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:57.789 INFO:teuthology.orchestra.run.smithi089.stdout:498216206351 2024-04-13T07:45:58.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:57 smithi203 bash[68060]: audit 2024-04-13T07:45:57.075376+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:58.048 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:57 smithi089 bash[84777]: audit 2024-04-13T07:45:57.075376+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:58.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:57 smithi089 bash[82331]: audit 2024-04-13T07:45:57.075376+0000 mon.a (mon.0) 602 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:45:58.987 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:58 smithi089 bash[84777]: cluster 2024-04-13T07:45:57.161761+0000 mgr.y (mgr.34107) 330 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:58.988 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:45:58 smithi089 bash[84777]: audit 2024-04-13T07:45:57.795374+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/1563399177' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:45:58.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:58 smithi089 bash[82331]: cluster 2024-04-13T07:45:57.161761+0000 mgr.y (mgr.34107) 330 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:58.988 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:45:58 smithi089 bash[82331]: audit 2024-04-13T07:45:57.795374+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/1563399177' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:45:59.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:58 smithi203 bash[68060]: cluster 2024-04-13T07:45:57.161761+0000 mgr.y (mgr.34107) 330 : cluster [DBG] pgmap v268: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:45:59.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:45:58 smithi203 bash[68060]: audit 2024-04-13T07:45:57.795374+0000 mon.c (mon.2) 25 : audit [DBG] from='client.? 172.21.15.89:0/1563399177' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2024-04-13T07:46:00.248 INFO:teuthology.orchestra.run.smithi089.stdout:433791696927 2024-04-13T07:46:01.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:00 smithi203 bash[68060]: cluster 2024-04-13T07:45:59.162647+0000 mgr.y (mgr.34107) 331 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:01.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:00 smithi203 bash[68060]: audit 2024-04-13T07:46:00.254165+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/3877811139' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:46:01.037 INFO:teuthology.orchestra.run.smithi089.stdout:455266533402 2024-04-13T07:46:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:00 smithi089 bash[82331]: cluster 2024-04-13T07:45:59.162647+0000 mgr.y (mgr.34107) 331 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:00 smithi089 bash[82331]: audit 2024-04-13T07:46:00.254165+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/3877811139' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:46:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:00 smithi089 bash[84777]: cluster 2024-04-13T07:45:59.162647+0000 mgr.y (mgr.34107) 331 : cluster [DBG] pgmap v269: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:01.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:00 smithi089 bash[84777]: audit 2024-04-13T07:46:00.254165+0000 mon.c (mon.2) 26 : audit [DBG] from='client.? 172.21.15.89:0/3877811139' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2024-04-13T07:46:01.113 INFO:tasks.cephadm.ceph_manager.ceph:need seq 498216206347 got 498216206351 for osd.4 2024-04-13T07:46:01.113 DEBUG:teuthology.parallel:result is None 2024-04-13T07:46:02.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:01 smithi089 bash[84777]: audit 2024-04-13T07:46:01.043732+0000 mon.a (mon.0) 603 : audit [DBG] from='client.? 172.21.15.89:0/3043628395' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:46:02.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:01 smithi089 bash[84777]: audit 2024-04-13T07:46:01.370821+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:46:02.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:01 smithi089 bash[82331]: audit 2024-04-13T07:46:01.043732+0000 mon.a (mon.0) 603 : audit [DBG] from='client.? 172.21.15.89:0/3043628395' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:46:02.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:01 smithi089 bash[82331]: audit 2024-04-13T07:46:01.370821+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:46:02.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:01 smithi203 bash[68060]: audit 2024-04-13T07:46:01.043732+0000 mon.a (mon.0) 603 : audit [DBG] from='client.? 172.21.15.89:0/3043628395' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2024-04-13T07:46:02.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:01 smithi203 bash[68060]: audit 2024-04-13T07:46:01.370821+0000 mon.a (mon.0) 604 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:46:02.766 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:46:02 smithi203 bash[60252]: ts=2024-04-13T07:46:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:46:02.926 INFO:teuthology.orchestra.run.smithi089.stdout:201863463133 2024-04-13T07:46:03.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:02 smithi089 bash[84777]: cluster 2024-04-13T07:46:01.163456+0000 mgr.y (mgr.34107) 332 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:03.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:02 smithi089 bash[82331]: cluster 2024-04-13T07:46:01.163456+0000 mgr.y (mgr.34107) 332 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:03.179 INFO:tasks.cephadm.ceph_manager.ceph:need seq 433791696924 got 433791696927 for osd.1 2024-04-13T07:46:03.179 DEBUG:teuthology.parallel:result is None 2024-04-13T07:46:03.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:02 smithi203 bash[68060]: cluster 2024-04-13T07:46:01.163456+0000 mgr.y (mgr.34107) 332 : cluster [DBG] pgmap v270: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:04.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:03 smithi089 bash[82331]: audit 2024-04-13T07:46:02.932696+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/2105673488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:46:04.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:03 smithi089 bash[84777]: audit 2024-04-13T07:46:02.932696+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/2105673488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:46:04.126 INFO:tasks.cephadm.ceph_manager.ceph:need seq 455266533398 got 455266533402 for osd.2 2024-04-13T07:46:04.126 DEBUG:teuthology.parallel:result is None 2024-04-13T07:46:04.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:03 smithi203 bash[68060]: audit 2024-04-13T07:46:02.932696+0000 mon.c (mon.2) 27 : audit [DBG] from='client.? 172.21.15.89:0/2105673488' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2024-04-13T07:46:04.319 INFO:teuthology.orchestra.run.smithi089.stdout:476741369878 2024-04-13T07:46:04.349 INFO:teuthology.orchestra.run.smithi089.stdout:236223201492 2024-04-13T07:46:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:04 smithi089 bash[84777]: cluster 2024-04-13T07:46:03.164103+0000 mgr.y (mgr.34107) 333 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:04 smithi089 bash[84777]: audit 2024-04-13T07:46:04.325097+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/2750182566' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:46:05.031 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:04 smithi089 bash[84777]: audit 2024-04-13T07:46:04.355705+0000 mon.a (mon.0) 605 : audit [DBG] from='client.? 172.21.15.89:0/3210223339' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:46:05.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:04 smithi089 bash[82331]: cluster 2024-04-13T07:46:03.164103+0000 mgr.y (mgr.34107) 333 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:05.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:04 smithi089 bash[82331]: audit 2024-04-13T07:46:04.325097+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/2750182566' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:46:05.032 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:04 smithi089 bash[82331]: audit 2024-04-13T07:46:04.355705+0000 mon.a (mon.0) 605 : audit [DBG] from='client.? 172.21.15.89:0/3210223339' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:46:05.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:04 smithi203 bash[68060]: cluster 2024-04-13T07:46:03.164103+0000 mgr.y (mgr.34107) 333 : cluster [DBG] pgmap v271: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:05.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:04 smithi203 bash[68060]: audit 2024-04-13T07:46:04.325097+0000 mon.c (mon.2) 28 : audit [DBG] from='client.? 172.21.15.89:0/2750182566' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2024-04-13T07:46:05.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:04 smithi203 bash[68060]: audit 2024-04-13T07:46:04.355705+0000 mon.a (mon.0) 605 : audit [DBG] from='client.? 172.21.15.89:0/3210223339' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2024-04-13T07:46:05.841 INFO:tasks.cephadm.ceph_manager.ceph:need seq 201863463129 got 201863463133 for osd.6 2024-04-13T07:46:05.841 DEBUG:teuthology.parallel:result is None 2024-04-13T07:46:05.994 INFO:teuthology.orchestra.run.smithi089.stdout:167503724777 2024-04-13T07:46:06.113 INFO:teuthology.orchestra.run.smithi089.stdout:412316860469 2024-04-13T07:46:06.126 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:46:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:46:05] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.43.0" 2024-04-13T07:46:06.176 INFO:tasks.cephadm.ceph_manager.ceph:need seq 476741369874 got 476741369878 for osd.3 2024-04-13T07:46:06.177 DEBUG:teuthology.parallel:result is None 2024-04-13T07:46:06.386 INFO:tasks.cephadm.ceph_manager.ceph:need seq 236223201487 got 236223201492 for osd.7 2024-04-13T07:46:06.386 DEBUG:teuthology.parallel:result is None 2024-04-13T07:46:07.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:06 smithi089 bash[82331]: cluster 2024-04-13T07:46:05.164823+0000 mgr.y (mgr.34107) 334 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:07.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:06 smithi089 bash[82331]: audit 2024-04-13T07:46:06.000442+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/2755012733' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:46:07.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:06 smithi089 bash[82331]: audit 2024-04-13T07:46:06.119171+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/877479252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:46:07.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:06 smithi089 bash[84777]: cluster 2024-04-13T07:46:05.164823+0000 mgr.y (mgr.34107) 334 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:07.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:06 smithi089 bash[84777]: audit 2024-04-13T07:46:06.000442+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/2755012733' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:46:07.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:06 smithi089 bash[84777]: audit 2024-04-13T07:46:06.119171+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/877479252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:46:07.154 INFO:tasks.cephadm.ceph_manager.ceph:need seq 412316860465 got 412316860469 for osd.0 2024-04-13T07:46:07.155 DEBUG:teuthology.parallel:result is None 2024-04-13T07:46:07.162 INFO:tasks.cephadm.ceph_manager.ceph:need seq 167503724773 got 167503724777 for osd.5 2024-04-13T07:46:07.162 DEBUG:teuthology.parallel:result is None 2024-04-13T07:46:07.162 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2024-04-13T07:46:07.163 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-13T07:46:07.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:06 smithi203 bash[68060]: cluster 2024-04-13T07:46:05.164823+0000 mgr.y (mgr.34107) 334 : cluster [DBG] pgmap v272: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:07.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:06 smithi203 bash[68060]: audit 2024-04-13T07:46:06.000442+0000 mon.c (mon.2) 29 : audit [DBG] from='client.? 172.21.15.89:0/2755012733' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2024-04-13T07:46:07.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:06 smithi203 bash[68060]: audit 2024-04-13T07:46:06.119171+0000 mon.c (mon.2) 30 : audit [DBG] from='client.? 172.21.15.89:0/877479252' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2024-04-13T07:46:08.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:08 smithi203 bash[68060]: cluster 2024-04-13T07:46:07.165510+0000 mgr.y (mgr.34107) 335 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:08.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:08 smithi203 bash[68060]: audit 2024-04-13T07:46:07.464655+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:08.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:08 smithi203 bash[68060]: audit 2024-04-13T07:46:07.476176+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:08.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:08 smithi089 bash[84777]: cluster 2024-04-13T07:46:07.165510+0000 mgr.y (mgr.34107) 335 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:08.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:08 smithi089 bash[84777]: audit 2024-04-13T07:46:07.464655+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:08.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:08 smithi089 bash[84777]: audit 2024-04-13T07:46:07.476176+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:08.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:08 smithi089 bash[82331]: cluster 2024-04-13T07:46:07.165510+0000 mgr.y (mgr.34107) 335 : cluster [DBG] pgmap v273: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:08.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:08 smithi089 bash[82331]: audit 2024-04-13T07:46:07.464655+0000 mon.a (mon.0) 606 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:08.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:08 smithi089 bash[82331]: audit 2024-04-13T07:46:07.476176+0000 mon.a (mon.0) 607 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:10.766 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:10 smithi203 bash[68060]: cluster 2024-04-13T07:46:09.166367+0000 mgr.y (mgr.34107) 336 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:10.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:10 smithi089 bash[84777]: cluster 2024-04-13T07:46:09.166367+0000 mgr.y (mgr.34107) 336 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:10.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:10 smithi089 bash[82331]: cluster 2024-04-13T07:46:09.166367+0000 mgr.y (mgr.34107) 336 : cluster [DBG] pgmap v274: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:11.938 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:46:11.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[84777]: audit 2024-04-13T07:46:10.691955+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:11.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[84777]: audit 2024-04-13T07:46:10.698855+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:11.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[84777]: audit 2024-04-13T07:46:11.268950+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:46:11.958 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[84777]: audit 2024-04-13T07:46:11.270498+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:46:11.959 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[84777]: audit 2024-04-13T07:46:11.278636+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:11.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[82331]: audit 2024-04-13T07:46:10.691955+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:11.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[82331]: audit 2024-04-13T07:46:10.698855+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:11.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[82331]: audit 2024-04-13T07:46:11.268950+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:46:11.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[82331]: audit 2024-04-13T07:46:11.270498+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:46:11.959 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:11 smithi089 bash[82331]: audit 2024-04-13T07:46:11.278636+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:12.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:11 smithi203 bash[68060]: audit 2024-04-13T07:46:10.691955+0000 mon.a (mon.0) 608 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:12.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:11 smithi203 bash[68060]: audit 2024-04-13T07:46:10.698855+0000 mon.a (mon.0) 609 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:12.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:11 smithi203 bash[68060]: audit 2024-04-13T07:46:11.268950+0000 mon.a (mon.0) 610 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:46:12.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:11 smithi203 bash[68060]: audit 2024-04-13T07:46:11.270498+0000 mon.a (mon.0) 611 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:46:12.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:11 smithi203 bash[68060]: audit 2024-04-13T07:46:11.278636+0000 mon.a (mon.0) 612 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:46:12.697 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:46:12 smithi203 bash[60252]: ts=2024-04-13T07:46:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:46:12.985 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:12 smithi089 bash[84777]: cluster 2024-04-13T07:46:11.167042+0000 mgr.y (mgr.34107) 337 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:12.986 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:12 smithi089 bash[84777]: audit 2024-04-13T07:46:12.076611+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:12.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:12 smithi089 bash[82331]: cluster 2024-04-13T07:46:11.167042+0000 mgr.y (mgr.34107) 337 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:12.986 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:12 smithi089 bash[82331]: audit 2024-04-13T07:46:12.076611+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:13.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:12 smithi203 bash[68060]: cluster 2024-04-13T07:46:11.167042+0000 mgr.y (mgr.34107) 337 : cluster [DBG] pgmap v275: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:13.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:12 smithi203 bash[68060]: audit 2024-04-13T07:46:12.076611+0000 mon.a (mon.0) 613 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:14.696 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:46:14.698 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-04-13T07:46:15.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:14 smithi203 bash[68060]: cluster 2024-04-13T07:46:13.168012+0000 mgr.y (mgr.34107) 338 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:15.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:14 smithi089 bash[82331]: cluster 2024-04-13T07:46:13.168012+0000 mgr.y (mgr.34107) 338 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:15.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:14 smithi089 bash[84777]: cluster 2024-04-13T07:46:13.168012+0000 mgr.y (mgr.34107) 338 : cluster [DBG] pgmap v276: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:15.415 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":276,"stamp":"2024-04-13T07:46:13.167274+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6559,"num_read_kb":13024,"num_write":1146,"num_write_kb":15064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1198,"ondisk_log_size":1198,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":259504,"kb_used_data":24236,"kb_used_omap":41,"kb_used_meta":235094,"kb_avail":749701712,"statfs":{"total":767960285184,"available":767694553088,"internally_reserved":0,"allocated":24817664,"data_stored":19882308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42533,"internal_metadata":240736731},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004320"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.372631+0000","last_change":"2024-04-13T07:44:53.372631+0000","last_active":"2024-04-13T07:44:53.372631+0000","last_peered":"2024-04-13T07:44:53.372631+0000","last_clean":"2024-04-13T07:44:53.372631+0000","last_became_active":"2024-04-13T07:44:53.371713+0000","last_became_peered":"2024-04-13T07:44:53.371713+0000","last_unstale":"2024-04-13T07:44:53.372631+0000","last_undegraded":"2024-04-13T07:44:53.372631+0000","last_fullsized":"2024-04-13T07:44:53.372631+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:10:27.491280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.349302+0000","last_change":"2024-04-13T07:43:56.606964+0000","last_active":"2024-04-13T07:44:53.349302+0000","last_peered":"2024-04-13T07:44:53.349302+0000","last_clean":"2024-04-13T07:44:53.349302+0000","last_became_active":"2024-04-13T07:43:56.606004+0000","last_became_peered":"2024-04-13T07:43:56.606004+0000","last_unstale":"2024-04-13T07:44:53.349302+0000","last_undegraded":"2024-04-13T07:44:53.349302+0000","last_fullsized":"2024-04-13T07:44:53.349302+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:21:34.212932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":104,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345907+0000","last_change":"2024-04-13T07:44:25.090311+0000","last_active":"2024-04-13T07:44:53.345907+0000","last_peered":"2024-04-13T07:44:53.345907+0000","last_clean":"2024-04-13T07:44:53.345907+0000","last_became_active":"2024-04-13T07:44:25.089751+0000","last_became_peered":"2024-04-13T07:44:25.089751+0000","last_unstale":"2024-04-13T07:44:53.345907+0000","last_undegraded":"2024-04-13T07:44:53.345907+0000","last_fullsized":"2024-04-13T07:44:53.345907+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:54:36.927258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":93,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.346118+0000","last_change":"2024-04-13T07:44:25.094249+0000","last_active":"2024-04-13T07:44:53.346118+0000","last_peered":"2024-04-13T07:44:53.346118+0000","last_clean":"2024-04-13T07:44:53.346118+0000","last_became_active":"2024-04-13T07:44:25.093610+0000","last_became_peered":"2024-04-13T07:44:25.093610+0000","last_unstale":"2024-04-13T07:44:53.346118+0000","last_undegraded":"2024-04-13T07:44:53.346118+0000","last_fullsized":"2024-04-13T07:44:53.346118+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:10:40.159024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.364318+0000","last_change":"2024-04-13T07:44:53.364318+0000","last_active":"2024-04-13T07:44:53.364318+0000","last_peered":"2024-04-13T07:44:53.364318+0000","last_clean":"2024-04-13T07:44:53.364318+0000","last_became_active":"2024-04-13T07:44:53.363461+0000","last_became_peered":"2024-04-13T07:44:53.363461+0000","last_unstale":"2024-04-13T07:44:53.364318+0000","last_undegraded":"2024-04-13T07:44:53.364318+0000","last_fullsized":"2024-04-13T07:44:53.364318+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:29:49.526269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":85,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.375884+0000","last_change":"2024-04-13T07:44:53.375884+0000","last_active":"2024-04-13T07:44:53.375884+0000","last_peered":"2024-04-13T07:44:53.375884+0000","last_clean":"2024-04-13T07:44:53.375884+0000","last_became_active":"2024-04-13T07:44:53.374876+0000","last_became_peered":"2024-04-13T07:44:53.374876+0000","last_unstale":"2024-04-13T07:44:53.375884+0000","last_undegraded":"2024-04-13T07:44:53.375884+0000","last_fullsized":"2024-04-13T07:44:53.375884+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:28:37.948151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.383176+0000","last_change":"2024-04-13T07:44:53.383176+0000","last_active":"2024-04-13T07:44:53.383176+0000","last_peered":"2024-04-13T07:44:53.383176+0000","last_clean":"2024-04-13T07:44:53.383176+0000","last_became_active":"2024-04-13T07:44:53.381266+0000","last_became_peered":"2024-04-13T07:44:53.381266+0000","last_unstale":"2024-04-13T07:44:53.383176+0000","last_undegraded":"2024-04-13T07:44:53.383176+0000","last_fullsized":"2024-04-13T07:44:53.383176+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:55:12.355011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":100,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.370062+0000","last_change":"2024-04-13T07:44:53.370062+0000","last_active":"2024-04-13T07:44:53.370062+0000","last_peered":"2024-04-13T07:44:53.370062+0000","last_clean":"2024-04-13T07:44:53.370062+0000","last_became_active":"2024-04-13T07:44:53.368994+0000","last_became_peered":"2024-04-13T07:44:53.368994+0000","last_unstale":"2024-04-13T07:44:53.370062+0000","last_undegraded":"2024-04-13T07:44:53.370062+0000","last_fullsized":"2024-04-13T07:44:53.370062+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:26:39.804415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":104,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.337135+0000","last_change":"2024-04-13T07:44:25.081714+0000","last_active":"2024-04-13T07:44:53.337135+0000","last_peered":"2024-04-13T07:44:53.337135+0000","last_clean":"2024-04-13T07:44:53.337135+0000","last_became_active":"2024-04-13T07:44:25.080865+0000","last_became_peered":"2024-04-13T07:44:25.080865+0000","last_unstale":"2024-04-13T07:44:53.337135+0000","last_undegraded":"2024-04-13T07:44:53.337135+0000","last_fullsized":"2024-04-13T07:44:53.337135+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:21:12.556095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.372891+0000","last_change":"2024-04-13T07:44:53.372891+0000","last_active":"2024-04-13T07:44:53.372891+0000","last_peered":"2024-04-13T07:44:53.372891+0000","last_clean":"2024-04-13T07:44:53.372891+0000","last_became_active":"2024-04-13T07:44:53.371865+0000","last_became_peered":"2024-04-13T07:44:53.371865+0000","last_unstale":"2024-04-13T07:44:53.372891+0000","last_undegraded":"2024-04-13T07:44:53.372891+0000","last_fullsized":"2024-04-13T07:44:53.372891+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:25:55.170651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.379949+0000","last_change":"2024-04-13T07:44:53.379949+0000","last_active":"2024-04-13T07:44:53.379949+0000","last_peered":"2024-04-13T07:44:53.379949+0000","last_clean":"2024-04-13T07:44:53.379949+0000","last_became_active":"2024-04-13T07:44:53.378955+0000","last_became_peered":"2024-04-13T07:44:53.378955+0000","last_unstale":"2024-04-13T07:44:53.379949+0000","last_undegraded":"2024-04-13T07:44:53.379949+0000","last_fullsized":"2024-04-13T07:44:53.379949+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:34:19.108316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.374084+0000","last_change":"2024-04-13T07:44:53.373062+0000","last_active":"2024-04-13T07:44:53.374084+0000","last_peered":"2024-04-13T07:44:53.374084+0000","last_clean":"2024-04-13T07:44:53.374084+0000","last_became_active":"2024-04-13T07:44:53.372071+0000","last_became_peered":"2024-04-13T07:44:53.372071+0000","last_unstale":"2024-04-13T07:44:53.374084+0000","last_undegraded":"2024-04-13T07:44:53.374084+0000","last_fullsized":"2024-04-13T07:44:53.374084+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:34:43.826639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"88'83","reported_seq":438,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348667+0000","last_change":"2024-04-13T07:43:56.597620+0000","last_active":"2024-04-13T07:44:53.348667+0000","last_peered":"2024-04-13T07:44:53.348667+0000","last_clean":"2024-04-13T07:44:53.348667+0000","last_became_active":"2024-04-13T07:43:56.596350+0000","last_became_peered":"2024-04-13T07:43:56.596350+0000","last_unstale":"2024-04-13T07:44:53.348667+0000","last_undegraded":"2024-04-13T07:44:53.348667+0000","last_fullsized":"2024-04-13T07:44:53.348667+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:01:41.693508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047849900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":578,"num_read_kb":3279,"num_write":60,"num_write_kb":2448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.343890+0000","last_change":"2024-04-13T07:44:25.089344+0000","last_active":"2024-04-13T07:44:53.343890+0000","last_peered":"2024-04-13T07:44:53.343890+0000","last_clean":"2024-04-13T07:44:53.343890+0000","last_became_active":"2024-04-13T07:44:25.088979+0000","last_became_peered":"2024-04-13T07:44:25.088979+0000","last_unstale":"2024-04-13T07:44:53.343890+0000","last_undegraded":"2024-04-13T07:44:53.343890+0000","last_fullsized":"2024-04-13T07:44:53.343890+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:59:10.043540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"88'90","reported_seq":394,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.385448+0000","last_change":"2024-04-13T07:44:53.382271+0000","last_active":"2024-04-13T07:44:53.385448+0000","last_peered":"2024-04-13T07:44:53.385448+0000","last_clean":"2024-04-13T07:44:53.385448+0000","last_became_active":"2024-04-13T07:44:53.379365+0000","last_became_peered":"2024-04-13T07:44:53.379365+0000","last_unstale":"2024-04-13T07:44:53.385448+0000","last_undegraded":"2024-04-13T07:44:53.385448+0000","last_fullsized":"2024-04-13T07:44:53.385448+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":90,"log_dups_size":0,"ondisk_log_size":90,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:57:54.866128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00093873800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":435,"num_read_kb":340,"num_write":60,"num_write_kb":1053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":85,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.346531+0000","last_change":"2024-04-13T07:44:25.092886+0000","last_active":"2024-04-13T07:44:53.346531+0000","last_peered":"2024-04-13T07:44:53.346531+0000","last_clean":"2024-04-13T07:44:53.346531+0000","last_became_active":"2024-04-13T07:44:25.091873+0000","last_became_peered":"2024-04-13T07:44:25.091873+0000","last_unstale":"2024-04-13T07:44:53.346531+0000","last_undegraded":"2024-04-13T07:44:53.346531+0000","last_fullsized":"2024-04-13T07:44:53.346531+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:52:04.470009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.353808+0000","last_change":"2024-04-13T07:41:47.256040+0000","last_active":"2024-04-13T07:44:53.353808+0000","last_peered":"2024-04-13T07:44:53.353808+0000","last_clean":"2024-04-13T07:44:53.353808+0000","last_became_active":"2024-04-13T07:41:47.255668+0000","last_became_peered":"2024-04-13T07:41:47.255668+0000","last_unstale":"2024-04-13T07:44:53.353808+0000","last_undegraded":"2024-04-13T07:44:53.353808+0000","last_fullsized":"2024-04-13T07:44:53.353808+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:18:17.880139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":62,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.354581+0000","last_change":"2024-04-13T07:43:56.617839+0000","last_active":"2024-04-13T07:44:53.354581+0000","last_peered":"2024-04-13T07:44:53.354581+0000","last_clean":"2024-04-13T07:44:53.354581+0000","last_became_active":"2024-04-13T07:43:56.615696+0000","last_became_peered":"2024-04-13T07:43:56.615696+0000","last_unstale":"2024-04-13T07:44:53.354581+0000","last_undegraded":"2024-04-13T07:44:53.354581+0000","last_fullsized":"2024-04-13T07:44:53.354581+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:19:56.651845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"88'55","reported_seq":364,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.373856+0000","last_change":"2024-04-13T07:44:53.372939+0000","last_active":"2024-04-13T07:44:53.373856+0000","last_peered":"2024-04-13T07:44:53.373856+0000","last_clean":"2024-04-13T07:44:53.373856+0000","last_became_active":"2024-04-13T07:44:53.371733+0000","last_became_peered":"2024-04-13T07:44:53.371733+0000","last_unstale":"2024-04-13T07:44:53.373856+0000","last_undegraded":"2024-04-13T07:44:53.373856+0000","last_fullsized":"2024-04-13T07:44:53.373856+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":55,"log_dups_size":0,"ondisk_log_size":55,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:51:32.511959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016787200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":368,"num_read_kb":474,"num_write":36,"num_write_kb":97,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":82,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.352239+0000","last_change":"2024-04-13T07:43:56.601972+0000","last_active":"2024-04-13T07:44:53.352239+0000","last_peered":"2024-04-13T07:44:53.352239+0000","last_clean":"2024-04-13T07:44:53.352239+0000","last_became_active":"2024-04-13T07:43:56.600913+0000","last_became_peered":"2024-04-13T07:43:56.600913+0000","last_unstale":"2024-04-13T07:44:53.352239+0000","last_undegraded":"2024-04-13T07:44:53.352239+0000","last_fullsized":"2024-04-13T07:44:53.352239+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:14:10.946781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":81,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.371051+0000","last_change":"2024-04-13T07:44:53.370164+0000","last_active":"2024-04-13T07:44:53.371051+0000","last_peered":"2024-04-13T07:44:53.371051+0000","last_clean":"2024-04-13T07:44:53.371051+0000","last_became_active":"2024-04-13T07:44:53.369333+0000","last_became_peered":"2024-04-13T07:44:53.369333+0000","last_unstale":"2024-04-13T07:44:53.371051+0000","last_undegraded":"2024-04-13T07:44:53.371051+0000","last_fullsized":"2024-04-13T07:44:53.371051+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:18:23.360881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"88'128","reported_seq":532,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.339728+0000","last_change":"2024-04-13T07:43:31.632113+0000","last_active":"2024-04-13T07:44:52.339728+0000","last_peered":"2024-04-13T07:44:52.339728+0000","last_clean":"2024-04-13T07:44:52.339728+0000","last_became_active":"2024-04-13T07:43:31.630264+0000","last_became_peered":"2024-04-13T07:43:31.630264+0000","last_unstale":"2024-04-13T07:44:52.339728+0000","last_undegraded":"2024-04-13T07:44:52.339728+0000","last_fullsized":"2024-04-13T07:44:52.339728+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:20:12.523379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0025084870000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":869,"num_read_kb":1763,"num_write":124,"num_write_kb":1097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.382584+0000","last_change":"2024-04-13T07:44:53.382584+0000","last_active":"2024-04-13T07:44:53.382584+0000","last_peered":"2024-04-13T07:44:53.382584+0000","last_clean":"2024-04-13T07:44:53.382584+0000","last_became_active":"2024-04-13T07:44:53.381694+0000","last_became_peered":"2024-04-13T07:44:53.381694+0000","last_unstale":"2024-04-13T07:44:53.382584+0000","last_undegraded":"2024-04-13T07:44:53.382584+0000","last_fullsized":"2024-04-13T07:44:53.382584+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:59:30.257522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.343250+0000","last_change":"2024-04-13T07:44:25.095413+0000","last_active":"2024-04-13T07:44:53.343250+0000","last_peered":"2024-04-13T07:44:53.343250+0000","last_clean":"2024-04-13T07:44:53.343250+0000","last_became_active":"2024-04-13T07:44:25.094703+0000","last_became_peered":"2024-04-13T07:44:25.094703+0000","last_unstale":"2024-04-13T07:44:53.343250+0000","last_undegraded":"2024-04-13T07:44:53.343250+0000","last_fullsized":"2024-04-13T07:44:53.343250+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:30:06.290854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"92'161","reported_seq":451,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.339221+0000","last_change":"2024-04-13T07:41:47.249040+0000","last_active":"2024-04-13T07:44:53.339221+0000","last_peered":"2024-04-13T07:44:53.339221+0000","last_clean":"2024-04-13T07:44:53.339221+0000","last_became_active":"2024-04-13T07:41:47.246557+0000","last_became_peered":"2024-04-13T07:41:47.246557+0000","last_unstale":"2024-04-13T07:44:53.339221+0000","last_undegraded":"2024-04-13T07:44:53.339221+0000","last_fullsized":"2024-04-13T07:44:53.339221+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_clean_scrub_stamp":"2024-04-13T07:24:24.657602+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:57:26.656959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"88'208","reported_seq":647,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.344893+0000","last_change":"2024-04-13T07:43:56.597581+0000","last_active":"2024-04-13T07:44:53.344893+0000","last_peered":"2024-04-13T07:44:53.344893+0000","last_clean":"2024-04-13T07:44:53.344893+0000","last_became_active":"2024-04-13T07:43:56.596278+0000","last_became_peered":"2024-04-13T07:43:56.596278+0000","last_unstale":"2024-04-13T07:44:53.344893+0000","last_undegraded":"2024-04-13T07:44:53.344893+0000","last_fullsized":"2024-04-13T07:44:53.344893+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:28:00.889374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001771379,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":818,"num_read_kb":1743,"num_write":191,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.353311+0000","last_change":"2024-04-13T07:43:31.646159+0000","last_active":"2024-04-13T07:44:53.353311+0000","last_peered":"2024-04-13T07:44:53.353311+0000","last_clean":"2024-04-13T07:44:53.353311+0000","last_became_active":"2024-04-13T07:43:31.644959+0000","last_became_peered":"2024-04-13T07:43:31.644959+0000","last_unstale":"2024-04-13T07:44:53.353311+0000","last_undegraded":"2024-04-13T07:44:53.353311+0000","last_fullsized":"2024-04-13T07:44:53.353311+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:26:25.210099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":62,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348579+0000","last_change":"2024-04-13T07:43:56.598373+0000","last_active":"2024-04-13T07:44:53.348579+0000","last_peered":"2024-04-13T07:44:53.348579+0000","last_clean":"2024-04-13T07:44:53.348579+0000","last_became_active":"2024-04-13T07:43:56.597433+0000","last_became_peered":"2024-04-13T07:43:56.597433+0000","last_unstale":"2024-04-13T07:44:53.348579+0000","last_undegraded":"2024-04-13T07:44:53.348579+0000","last_fullsized":"2024-04-13T07:44:53.348579+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:22:35.235889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"88'141","reported_seq":672,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.337098+0000","last_change":"2024-04-13T07:44:25.071644+0000","last_active":"2024-04-13T07:44:53.337098+0000","last_peered":"2024-04-13T07:44:53.337098+0000","last_clean":"2024-04-13T07:44:53.337098+0000","last_became_active":"2024-04-13T07:44:25.069930+0000","last_became_peered":"2024-04-13T07:44:25.069930+0000","last_unstale":"2024-04-13T07:44:53.337098+0000","last_undegraded":"2024-04-13T07:44:53.337098+0000","last_fullsized":"2024-04-13T07:44:53.337098+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":141,"log_dups_size":0,"ondisk_log_size":141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:32:17.505794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064946999999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":967,"num_read_kb":766,"num_write":117,"num_write_kb":1090,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.353614+0000","last_change":"2024-04-13T07:44:25.084001+0000","last_active":"2024-04-13T07:44:53.353614+0000","last_peered":"2024-04-13T07:44:53.353614+0000","last_clean":"2024-04-13T07:44:53.353614+0000","last_became_active":"2024-04-13T07:44:25.080976+0000","last_became_peered":"2024-04-13T07:44:25.080976+0000","last_unstale":"2024-04-13T07:44:53.353614+0000","last_undegraded":"2024-04-13T07:44:53.353614+0000","last_fullsized":"2024-04-13T07:44:53.353614+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:46:58.772048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.379866+0000","last_change":"2024-04-13T07:44:53.379866+0000","last_active":"2024-04-13T07:44:53.379866+0000","last_peered":"2024-04-13T07:44:53.379866+0000","last_clean":"2024-04-13T07:44:53.379866+0000","last_became_active":"2024-04-13T07:44:53.378197+0000","last_became_peered":"2024-04-13T07:44:53.378197+0000","last_unstale":"2024-04-13T07:44:53.379866+0000","last_undegraded":"2024-04-13T07:44:53.379866+0000","last_fullsized":"2024-04-13T07:44:53.379866+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:16:15.536975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"88'69","reported_seq":490,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345319+0000","last_change":"2024-04-13T07:43:31.639857+0000","last_active":"2024-04-13T07:44:53.345319+0000","last_peered":"2024-04-13T07:44:53.345319+0000","last_clean":"2024-04-13T07:44:53.345319+0000","last_became_active":"2024-04-13T07:43:31.638318+0000","last_became_peered":"2024-04-13T07:43:31.638318+0000","last_unstale":"2024-04-13T07:44:53.345319+0000","last_undegraded":"2024-04-13T07:44:53.345319+0000","last_fullsized":"2024-04-13T07:44:53.345319+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:09:42.077772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078461199999999996,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":480,"num_read_kb":1879,"num_write":69,"num_write_kb":5358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.386265+0000","last_change":"2024-04-13T07:44:53.381489+0000","last_active":"2024-04-13T07:44:53.386265+0000","last_peered":"2024-04-13T07:44:53.386265+0000","last_clean":"2024-04-13T07:44:53.386265+0000","last_became_active":"2024-04-13T07:44:53.379441+0000","last_became_peered":"2024-04-13T07:44:53.379441+0000","last_unstale":"2024-04-13T07:44:53.386265+0000","last_undegraded":"2024-04-13T07:44:53.386265+0000","last_fullsized":"2024-04-13T07:44:53.386265+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:33:57.409956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":97,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.363372+0000","last_change":"2024-04-13T07:44:25.083905+0000","last_active":"2024-04-13T07:44:52.363372+0000","last_peered":"2024-04-13T07:44:52.363372+0000","last_clean":"2024-04-13T07:44:52.363372+0000","last_became_active":"2024-04-13T07:44:25.082900+0000","last_became_peered":"2024-04-13T07:44:25.082900+0000","last_unstale":"2024-04-13T07:44:52.363372+0000","last_undegraded":"2024-04-13T07:44:52.363372+0000","last_fullsized":"2024-04-13T07:44:52.363372+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:42:59.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"88'202","reported_seq":938,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.353027+0000","last_change":"2024-04-13T07:43:31.644563+0000","last_active":"2024-04-13T07:44:53.353027+0000","last_peered":"2024-04-13T07:44:53.353027+0000","last_clean":"2024-04-13T07:44:53.353027+0000","last_became_active":"2024-04-13T07:43:31.643395+0000","last_became_peered":"2024-04-13T07:43:31.643395+0000","last_unstale":"2024-04-13T07:44:53.353027+0000","last_undegraded":"2024-04-13T07:44:53.353027+0000","last_fullsized":"2024-04-13T07:44:53.353027+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:57:11.613549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010273330000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1479,"num_read_kb":1788,"num_write":189,"num_write_kb":330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":63,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.357356+0000","last_change":"2024-04-13T07:44:53.356007+0000","last_active":"2024-04-13T07:44:53.357356+0000","last_peered":"2024-04-13T07:44:53.357356+0000","last_clean":"2024-04-13T07:44:53.357356+0000","last_became_active":"2024-04-13T07:44:53.355044+0000","last_became_peered":"2024-04-13T07:44:53.355044+0000","last_unstale":"2024-04-13T07:44:53.357356+0000","last_undegraded":"2024-04-13T07:44:53.357356+0000","last_fullsized":"2024-04-13T07:44:53.357356+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:40:11.027657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.344148+0000","last_change":"2024-04-13T07:44:25.096741+0000","last_active":"2024-04-13T07:44:53.344148+0000","last_peered":"2024-04-13T07:44:53.344148+0000","last_clean":"2024-04-13T07:44:53.344148+0000","last_became_active":"2024-04-13T07:44:25.095703+0000","last_became_peered":"2024-04-13T07:44:25.095703+0000","last_unstale":"2024-04-13T07:44:53.344148+0000","last_undegraded":"2024-04-13T07:44:53.344148+0000","last_fullsized":"2024-04-13T07:44:53.344148+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:34:55.859791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.376771+0000","last_change":"2024-04-13T07:44:53.376771+0000","last_active":"2024-04-13T07:44:53.376771+0000","last_peered":"2024-04-13T07:44:53.376771+0000","last_clean":"2024-04-13T07:44:53.376771+0000","last_became_active":"2024-04-13T07:44:53.376030+0000","last_became_peered":"2024-04-13T07:44:53.376030+0000","last_unstale":"2024-04-13T07:44:53.376771+0000","last_undegraded":"2024-04-13T07:44:53.376771+0000","last_fullsized":"2024-04-13T07:44:53.376771+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:10:06.094615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.370151+0000","last_change":"2024-04-13T07:44:53.370151+0000","last_active":"2024-04-13T07:44:53.370151+0000","last_peered":"2024-04-13T07:44:53.370151+0000","last_clean":"2024-04-13T07:44:53.370151+0000","last_became_active":"2024-04-13T07:44:53.369019+0000","last_became_peered":"2024-04-13T07:44:53.369019+0000","last_unstale":"2024-04-13T07:44:53.370151+0000","last_undegraded":"2024-04-13T07:44:53.370151+0000","last_fullsized":"2024-04-13T07:44:53.370151+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:44:52.317749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.367871+0000","last_change":"2024-04-13T07:44:53.367871+0000","last_active":"2024-04-13T07:44:53.367871+0000","last_peered":"2024-04-13T07:44:53.367871+0000","last_clean":"2024-04-13T07:44:53.367871+0000","last_became_active":"2024-04-13T07:44:53.365445+0000","last_became_peered":"2024-04-13T07:44:53.365445+0000","last_unstale":"2024-04-13T07:44:53.367871+0000","last_undegraded":"2024-04-13T07:44:53.367871+0000","last_fullsized":"2024-04-13T07:44:53.367871+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:55:10.819547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":96,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.382864+0000","last_change":"2024-04-13T07:44:53.381760+0000","last_active":"2024-04-13T07:44:53.382864+0000","last_peered":"2024-04-13T07:44:53.382864+0000","last_clean":"2024-04-13T07:44:53.382864+0000","last_became_active":"2024-04-13T07:44:53.380871+0000","last_became_peered":"2024-04-13T07:44:53.380871+0000","last_unstale":"2024-04-13T07:44:53.382864+0000","last_undegraded":"2024-04-13T07:44:53.382864+0000","last_fullsized":"2024-04-13T07:44:53.382864+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:54:55.694927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.382451+0000","last_change":"2024-04-13T07:44:53.382451+0000","last_active":"2024-04-13T07:44:53.382451+0000","last_peered":"2024-04-13T07:44:53.382451+0000","last_clean":"2024-04-13T07:44:53.382451+0000","last_became_active":"2024-04-13T07:44:53.381304+0000","last_became_peered":"2024-04-13T07:44:53.381304+0000","last_unstale":"2024-04-13T07:44:53.382451+0000","last_undegraded":"2024-04-13T07:44:53.382451+0000","last_fullsized":"2024-04-13T07:44:53.382451+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:59:02.406363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":94,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.361823+0000","last_change":"2024-04-13T07:44:53.361823+0000","last_active":"2024-04-13T07:44:53.361823+0000","last_peered":"2024-04-13T07:44:53.361823+0000","last_clean":"2024-04-13T07:44:53.361823+0000","last_became_active":"2024-04-13T07:44:53.360817+0000","last_became_peered":"2024-04-13T07:44:53.360817+0000","last_unstale":"2024-04-13T07:44:53.361823+0000","last_undegraded":"2024-04-13T07:44:53.361823+0000","last_fullsized":"2024-04-13T07:44:53.361823+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:08:18.067533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":81,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.373053+0000","last_change":"2024-04-13T07:44:53.373053+0000","last_active":"2024-04-13T07:44:53.373053+0000","last_peered":"2024-04-13T07:44:53.373053+0000","last_clean":"2024-04-13T07:44:53.373053+0000","last_became_active":"2024-04-13T07:44:53.372124+0000","last_became_peered":"2024-04-13T07:44:53.372124+0000","last_unstale":"2024-04-13T07:44:53.373053+0000","last_undegraded":"2024-04-13T07:44:53.373053+0000","last_fullsized":"2024-04-13T07:44:53.373053+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:54:02.008646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":82,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348275+0000","last_change":"2024-04-13T07:44:25.089560+0000","last_active":"2024-04-13T07:44:53.348275+0000","last_peered":"2024-04-13T07:44:53.348275+0000","last_clean":"2024-04-13T07:44:53.348275+0000","last_became_active":"2024-04-13T07:44:25.088757+0000","last_became_peered":"2024-04-13T07:44:25.088757+0000","last_unstale":"2024-04-13T07:44:53.348275+0000","last_undegraded":"2024-04-13T07:44:53.348275+0000","last_fullsized":"2024-04-13T07:44:53.348275+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:03:49.823397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348200+0000","last_change":"2024-04-13T07:43:31.643631+0000","last_active":"2024-04-13T07:44:53.348200+0000","last_peered":"2024-04-13T07:44:53.348200+0000","last_clean":"2024-04-13T07:44:53.348200+0000","last_became_active":"2024-04-13T07:43:31.641747+0000","last_became_peered":"2024-04-13T07:43:31.641747+0000","last_unstale":"2024-04-13T07:44:53.348200+0000","last_undegraded":"2024-04-13T07:44:53.348200+0000","last_fullsized":"2024-04-13T07:44:53.348200+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:21:55.189042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":68,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345509+0000","last_change":"2024-04-13T07:32:03.078696+0000","last_active":"2024-04-13T07:44:53.345509+0000","last_peered":"2024-04-13T07:44:53.345509+0000","last_clean":"2024-04-13T07:44:53.345509+0000","last_became_active":"2024-04-13T07:32:03.077798+0000","last_became_peered":"2024-04-13T07:32:03.077798+0000","last_unstale":"2024-04-13T07:44:53.345509+0000","last_undegraded":"2024-04-13T07:44:53.345509+0000","last_fullsized":"2024-04-13T07:44:53.345509+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:32:02.676056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":102,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.384421+0000","last_change":"2024-04-13T07:44:53.384421+0000","last_active":"2024-04-13T07:44:53.384421+0000","last_peered":"2024-04-13T07:44:53.384421+0000","last_clean":"2024-04-13T07:44:53.384421+0000","last_became_active":"2024-04-13T07:44:53.383605+0000","last_became_peered":"2024-04-13T07:44:53.383605+0000","last_unstale":"2024-04-13T07:44:53.384421+0000","last_undegraded":"2024-04-13T07:44:53.384421+0000","last_fullsized":"2024-04-13T07:44:53.384421+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:31:47.009049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":100,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.364990+0000","last_change":"2024-04-13T07:44:53.364027+0000","last_active":"2024-04-13T07:44:53.364990+0000","last_peered":"2024-04-13T07:44:53.364990+0000","last_clean":"2024-04-13T07:44:53.364990+0000","last_became_active":"2024-04-13T07:44:53.362973+0000","last_became_peered":"2024-04-13T07:44:53.362973+0000","last_unstale":"2024-04-13T07:44:53.364990+0000","last_undegraded":"2024-04-13T07:44:53.364990+0000","last_fullsized":"2024-04-13T07:44:53.364990+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:54:20.558297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":66,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.346597+0000","last_change":"2024-04-13T07:32:03.141831+0000","last_active":"2024-04-13T07:44:53.346597+0000","last_peered":"2024-04-13T07:44:53.346597+0000","last_clean":"2024-04-13T07:44:53.346597+0000","last_became_active":"2024-04-13T07:32:03.141053+0000","last_became_peered":"2024-04-13T07:32:03.141053+0000","last_unstale":"2024-04-13T07:44:53.346597+0000","last_undegraded":"2024-04-13T07:44:53.346597+0000","last_fullsized":"2024-04-13T07:44:53.346597+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:24:08.376151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.365521+0000","last_change":"2024-04-13T07:44:53.364935+0000","last_active":"2024-04-13T07:44:53.365521+0000","last_peered":"2024-04-13T07:44:53.365521+0000","last_clean":"2024-04-13T07:44:53.365521+0000","last_became_active":"2024-04-13T07:44:53.364004+0000","last_became_peered":"2024-04-13T07:44:53.364004+0000","last_unstale":"2024-04-13T07:44:53.365521+0000","last_undegraded":"2024-04-13T07:44:53.365521+0000","last_fullsized":"2024-04-13T07:44:53.365521+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:07:08.723096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.346235+0000","last_change":"2024-04-13T07:44:25.083540+0000","last_active":"2024-04-13T07:44:53.346235+0000","last_peered":"2024-04-13T07:44:53.346235+0000","last_clean":"2024-04-13T07:44:53.346235+0000","last_became_active":"2024-04-13T07:44:25.079738+0000","last_became_peered":"2024-04-13T07:44:25.079738+0000","last_unstale":"2024-04-13T07:44:53.346235+0000","last_undegraded":"2024-04-13T07:44:53.346235+0000","last_fullsized":"2024-04-13T07:44:53.346235+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:52:46.929779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":97,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348493+0000","last_change":"2024-04-13T07:41:47.264342+0000","last_active":"2024-04-13T07:44:53.348493+0000","last_peered":"2024-04-13T07:44:53.348493+0000","last_clean":"2024-04-13T07:44:53.348493+0000","last_became_active":"2024-04-13T07:41:47.262850+0000","last_became_peered":"2024-04-13T07:41:47.262850+0000","last_unstale":"2024-04-13T07:44:53.348493+0000","last_undegraded":"2024-04-13T07:44:53.348493+0000","last_fullsized":"2024-04-13T07:44:53.348493+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:12:51.340350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.350362+0000","last_change":"2024-04-13T07:44:25.085323+0000","last_active":"2024-04-13T07:44:53.350362+0000","last_peered":"2024-04-13T07:44:53.350362+0000","last_clean":"2024-04-13T07:44:53.350362+0000","last_became_active":"2024-04-13T07:44:25.084440+0000","last_became_peered":"2024-04-13T07:44:25.084440+0000","last_unstale":"2024-04-13T07:44:53.350362+0000","last_undegraded":"2024-04-13T07:44:53.350362+0000","last_fullsized":"2024-04-13T07:44:53.350362+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:53:26.066053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.362437+0000","last_change":"2024-04-13T07:44:53.361144+0000","last_active":"2024-04-13T07:44:53.362437+0000","last_peered":"2024-04-13T07:44:53.362437+0000","last_clean":"2024-04-13T07:44:53.362437+0000","last_became_active":"2024-04-13T07:44:53.360244+0000","last_became_peered":"2024-04-13T07:44:53.360244+0000","last_unstale":"2024-04-13T07:44:53.362437+0000","last_undegraded":"2024-04-13T07:44:53.362437+0000","last_fullsized":"2024-04-13T07:44:53.362437+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:39:43.030717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":80,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.363137+0000","last_change":"2024-04-13T07:44:25.083258+0000","last_active":"2024-04-13T07:44:52.363137+0000","last_peered":"2024-04-13T07:44:52.363137+0000","last_clean":"2024-04-13T07:44:52.363137+0000","last_became_active":"2024-04-13T07:44:25.082364+0000","last_became_peered":"2024-04-13T07:44:25.082364+0000","last_unstale":"2024-04-13T07:44:52.363137+0000","last_undegraded":"2024-04-13T07:44:52.363137+0000","last_fullsized":"2024-04-13T07:44:52.363137+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:24:05.070684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348766+0000","last_change":"2024-04-13T07:44:25.081436+0000","last_active":"2024-04-13T07:44:53.348766+0000","last_peered":"2024-04-13T07:44:53.348766+0000","last_clean":"2024-04-13T07:44:53.348766+0000","last_became_active":"2024-04-13T07:44:25.080738+0000","last_became_peered":"2024-04-13T07:44:25.080738+0000","last_unstale":"2024-04-13T07:44:53.348766+0000","last_undegraded":"2024-04-13T07:44:53.348766+0000","last_fullsized":"2024-04-13T07:44:53.348766+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:01:13.465690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345736+0000","last_change":"2024-04-13T07:44:25.097395+0000","last_active":"2024-04-13T07:44:53.345736+0000","last_peered":"2024-04-13T07:44:53.345736+0000","last_clean":"2024-04-13T07:44:53.345736+0000","last_became_active":"2024-04-13T07:44:25.096490+0000","last_became_peered":"2024-04-13T07:44:25.096490+0000","last_unstale":"2024-04-13T07:44:53.345736+0000","last_undegraded":"2024-04-13T07:44:53.345736+0000","last_fullsized":"2024-04-13T07:44:53.345736+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:40:03.850170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":100,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.361793+0000","last_change":"2024-04-13T07:44:53.360987+0000","last_active":"2024-04-13T07:44:53.361793+0000","last_peered":"2024-04-13T07:44:53.361793+0000","last_clean":"2024-04-13T07:44:53.361793+0000","last_became_active":"2024-04-13T07:44:53.359750+0000","last_became_peered":"2024-04-13T07:44:53.359750+0000","last_unstale":"2024-04-13T07:44:53.361793+0000","last_undegraded":"2024-04-13T07:44:53.361793+0000","last_fullsized":"2024-04-13T07:44:53.361793+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:28:14.428704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":77,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.377475+0000","last_change":"2024-04-13T07:44:53.377475+0000","last_active":"2024-04-13T07:44:53.377475+0000","last_peered":"2024-04-13T07:44:53.377475+0000","last_clean":"2024-04-13T07:44:53.377475+0000","last_became_active":"2024-04-13T07:44:53.374822+0000","last_became_peered":"2024-04-13T07:44:53.374822+0000","last_unstale":"2024-04-13T07:44:53.377475+0000","last_undegraded":"2024-04-13T07:44:53.377475+0000","last_fullsized":"2024-04-13T07:44:53.377475+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:43:35.887419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"92'23","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.379615+0000","last_change":"2024-04-13T07:44:53.379615+0000","last_active":"2024-04-13T07:44:53.379615+0000","last_peered":"2024-04-13T07:44:53.379615+0000","last_clean":"2024-04-13T07:44:53.379615+0000","last_became_active":"2024-04-13T07:44:53.378060+0000","last_became_peered":"2024-04-13T07:44:53.378060+0000","last_unstale":"2024-04-13T07:44:53.379615+0000","last_undegraded":"2024-04-13T07:44:53.379615+0000","last_fullsized":"2024-04-13T07:44:53.379615+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:59:57.314373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.352186+0000","last_change":"2024-04-13T07:43:56.598624+0000","last_active":"2024-04-13T07:44:53.352186+0000","last_peered":"2024-04-13T07:44:53.352186+0000","last_clean":"2024-04-13T07:44:53.352186+0000","last_became_active":"2024-04-13T07:43:56.597710+0000","last_became_peered":"2024-04-13T07:43:56.597710+0000","last_unstale":"2024-04-13T07:44:53.352186+0000","last_undegraded":"2024-04-13T07:44:53.352186+0000","last_fullsized":"2024-04-13T07:44:53.352186+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:47:49.807918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.370798+0000","last_change":"2024-04-13T07:44:53.370798+0000","last_active":"2024-04-13T07:44:53.370798+0000","last_peered":"2024-04-13T07:44:53.370798+0000","last_clean":"2024-04-13T07:44:53.370798+0000","last_became_active":"2024-04-13T07:44:53.369966+0000","last_became_peered":"2024-04-13T07:44:53.369966+0000","last_unstale":"2024-04-13T07:44:53.370798+0000","last_undegraded":"2024-04-13T07:44:53.370798+0000","last_fullsized":"2024-04-13T07:44:53.370798+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:02:25.087869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":33,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345607+0000","last_change":"2024-04-13T07:44:25.094541+0000","last_active":"2024-04-13T07:44:53.345607+0000","last_peered":"2024-04-13T07:44:53.345607+0000","last_clean":"2024-04-13T07:44:53.345607+0000","last_became_active":"2024-04-13T07:44:25.093753+0000","last_became_peered":"2024-04-13T07:44:25.093753+0000","last_unstale":"2024-04-13T07:44:53.345607+0000","last_undegraded":"2024-04-13T07:44:53.345607+0000","last_fullsized":"2024-04-13T07:44:53.345607+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:19:45.907198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":81,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.360540+0000","last_change":"2024-04-13T07:43:31.636805+0000","last_active":"2024-04-13T07:44:52.360540+0000","last_peered":"2024-04-13T07:44:52.360540+0000","last_clean":"2024-04-13T07:44:52.360540+0000","last_became_active":"2024-04-13T07:43:31.635810+0000","last_became_peered":"2024-04-13T07:43:31.635810+0000","last_unstale":"2024-04-13T07:44:52.360540+0000","last_undegraded":"2024-04-13T07:44:52.360540+0000","last_fullsized":"2024-04-13T07:44:52.360540+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:36:07.875306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.343004+0000","last_change":"2024-04-13T07:44:25.097296+0000","last_active":"2024-04-13T07:44:53.343004+0000","last_peered":"2024-04-13T07:44:53.343004+0000","last_clean":"2024-04-13T07:44:53.343004+0000","last_became_active":"2024-04-13T07:44:25.096518+0000","last_became_peered":"2024-04-13T07:44:25.096518+0000","last_unstale":"2024-04-13T07:44:53.343004+0000","last_undegraded":"2024-04-13T07:44:53.343004+0000","last_fullsized":"2024-04-13T07:44:53.343004+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:35:13.762142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.349457+0000","last_change":"2024-04-13T07:44:25.083097+0000","last_active":"2024-04-13T07:44:53.349457+0000","last_peered":"2024-04-13T07:44:53.349457+0000","last_clean":"2024-04-13T07:44:53.349457+0000","last_became_active":"2024-04-13T07:44:25.082218+0000","last_became_peered":"2024-04-13T07:44:25.082218+0000","last_unstale":"2024-04-13T07:44:53.349457+0000","last_undegraded":"2024-04-13T07:44:53.349457+0000","last_fullsized":"2024-04-13T07:44:53.349457+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:23:08.217840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":80,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.356724+0000","last_change":"2024-04-13T07:43:31.635070+0000","last_active":"2024-04-13T07:44:52.356724+0000","last_peered":"2024-04-13T07:44:52.356724+0000","last_clean":"2024-04-13T07:44:52.356724+0000","last_became_active":"2024-04-13T07:43:31.634074+0000","last_became_peered":"2024-04-13T07:43:31.634074+0000","last_unstale":"2024-04-13T07:44:52.356724+0000","last_undegraded":"2024-04-13T07:44:52.356724+0000","last_fullsized":"2024-04-13T07:44:52.356724+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:16:42.369895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.343599+0000","last_change":"2024-04-13T07:44:25.089335+0000","last_active":"2024-04-13T07:44:53.343599+0000","last_peered":"2024-04-13T07:44:53.343599+0000","last_clean":"2024-04-13T07:44:53.343599+0000","last_became_active":"2024-04-13T07:44:25.088925+0000","last_became_peered":"2024-04-13T07:44:25.088925+0000","last_unstale":"2024-04-13T07:44:53.343599+0000","last_undegraded":"2024-04-13T07:44:53.343599+0000","last_fullsized":"2024-04-13T07:44:53.343599+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:42:45.826951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.350745+0000","last_change":"2024-04-13T07:44:25.086154+0000","last_active":"2024-04-13T07:44:53.350745+0000","last_peered":"2024-04-13T07:44:53.350745+0000","last_clean":"2024-04-13T07:44:53.350745+0000","last_became_active":"2024-04-13T07:44:25.085727+0000","last_became_peered":"2024-04-13T07:44:25.085727+0000","last_unstale":"2024-04-13T07:44:53.350745+0000","last_undegraded":"2024-04-13T07:44:53.350745+0000","last_fullsized":"2024-04-13T07:44:53.350745+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:20:24.586059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":71,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.385363+0000","last_change":"2024-04-13T07:44:53.381154+0000","last_active":"2024-04-13T07:44:53.385363+0000","last_peered":"2024-04-13T07:44:53.385363+0000","last_clean":"2024-04-13T07:44:53.385363+0000","last_became_active":"2024-04-13T07:44:53.379300+0000","last_became_peered":"2024-04-13T07:44:53.379300+0000","last_unstale":"2024-04-13T07:44:53.385363+0000","last_undegraded":"2024-04-13T07:44:53.385363+0000","last_fullsized":"2024-04-13T07:44:53.385363+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:22:35.755043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":107,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.380849+0000","last_change":"2024-04-13T07:44:53.380849+0000","last_active":"2024-04-13T07:44:53.380849+0000","last_peered":"2024-04-13T07:44:53.380849+0000","last_clean":"2024-04-13T07:44:53.380849+0000","last_became_active":"2024-04-13T07:44:53.380000+0000","last_became_peered":"2024-04-13T07:44:53.380000+0000","last_unstale":"2024-04-13T07:44:53.380849+0000","last_undegraded":"2024-04-13T07:44:53.380849+0000","last_fullsized":"2024-04-13T07:44:53.380849+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:19:53.653413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":100,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.368700+0000","last_change":"2024-04-13T07:44:53.368700+0000","last_active":"2024-04-13T07:44:53.368700+0000","last_peered":"2024-04-13T07:44:53.368700+0000","last_clean":"2024-04-13T07:44:53.368700+0000","last_became_active":"2024-04-13T07:44:53.365796+0000","last_became_peered":"2024-04-13T07:44:53.365796+0000","last_unstale":"2024-04-13T07:44:53.368700+0000","last_undegraded":"2024-04-13T07:44:53.368700+0000","last_fullsized":"2024-04-13T07:44:53.368700+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:35:41.555072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":65,"num_read_kb":68,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24245,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12032,"num_write":846,"num_write_kb":12618,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5545,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201494,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37084,"kb_used_data":1872,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93708068,"statfs":{"total":95995035648,"available":95957061632,"internally_reserved":0,"allocated":1916928,"data_stored":1284396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":36043211},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463136,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32972,"kb_used_data":5952,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93712180,"statfs":{"total":95995035648,"available":95961272320,"internally_reserved":0,"allocated":6094848,"data_stored":5477969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":101,"seq":433791696930,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37056,"kb_used_data":5364,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708096,"statfs":{"total":95995035648,"available":95957090304,"internally_reserved":0,"allocated":5492736,"data_stored":4883391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7566,"internal_metadata":32432754},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":96,"seq":412316860471,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29392,"kb_used_data":1852,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93715760,"statfs":{"total":95995035648,"available":95964938240,"internally_reserved":0,"allocated":1896448,"data_stored":1280040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8046,"internal_metadata":28172434},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":106,"seq":455266533405,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28812,"kb_used_data":1272,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716340,"statfs":{"total":95995035648,"available":95965532160,"internally_reserved":0,"allocated":1302528,"data_stored":689684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8387,"internal_metadata":28172093},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":111,"seq":476741369880,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32968,"kb_used_data":1268,"kb_used_omap":6,"kb_used_meta":31673,"kb_avail":93712184,"statfs":{"total":95995035648,"available":95961276416,"internally_reserved":0,"allocated":1298432,"data_stored":689158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6212,"internal_metadata":32434108},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":116,"seq":498216206354,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28828,"kb_used_data":1288,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716324,"statfs":{"total":95995035648,"available":95965515776,"internally_reserved":0,"allocated":1318912,"data_stored":693970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7546,"internal_metadata":28172934},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724779,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32392,"kb_used_data":5368,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93712760,"statfs":{"total":95995035648,"available":95961866240,"internally_reserved":0,"allocated":5496832,"data_stored":4883700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4242,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5972,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2552,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3317,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5952,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-13T07:46:15.420 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph pg dump --format=json 2024-04-13T07:46:16.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:15 smithi203 bash[68060]: audit 2024-04-13T07:46:14.684184+0000 mgr.y (mgr.34107) 339 : audit [DBG] from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:46:16.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:46:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:46:15] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:46:16.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:15 smithi089 bash[84777]: audit 2024-04-13T07:46:14.684184+0000 mgr.y (mgr.34107) 339 : audit [DBG] from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:46:16.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:15 smithi089 bash[82331]: audit 2024-04-13T07:46:14.684184+0000 mgr.y (mgr.34107) 339 : audit [DBG] from='client.34533 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:46:17.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:16 smithi203 bash[68060]: cluster 2024-04-13T07:46:15.168725+0000 mgr.y (mgr.34107) 340 : cluster [DBG] pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:17.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:16 smithi089 bash[84777]: cluster 2024-04-13T07:46:15.168725+0000 mgr.y (mgr.34107) 340 : cluster [DBG] pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:17.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:16 smithi089 bash[82331]: cluster 2024-04-13T07:46:15.168725+0000 mgr.y (mgr.34107) 340 : cluster [DBG] pgmap v277: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:19.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:18 smithi203 bash[68060]: cluster 2024-04-13T07:46:17.169550+0000 mgr.y (mgr.34107) 341 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:19.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:18 smithi089 bash[82331]: cluster 2024-04-13T07:46:17.169550+0000 mgr.y (mgr.34107) 341 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:19.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:18 smithi089 bash[84777]: cluster 2024-04-13T07:46:17.169550+0000 mgr.y (mgr.34107) 341 : cluster [DBG] pgmap v278: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:20.161 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:46:21.005 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:20 smithi089 bash[82331]: cluster 2024-04-13T07:46:19.170453+0000 mgr.y (mgr.34107) 342 : cluster [DBG] pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:21.005 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:20 smithi089 bash[84777]: cluster 2024-04-13T07:46:19.170453+0000 mgr.y (mgr.34107) 342 : cluster [DBG] pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:21.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:20 smithi203 bash[68060]: cluster 2024-04-13T07:46:19.170453+0000 mgr.y (mgr.34107) 342 : cluster [DBG] pgmap v279: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:22.724 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:46:22 smithi203 bash[60252]: ts=2024-04-13T07:46:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:46:22.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:22 smithi089 bash[84777]: cluster 2024-04-13T07:46:21.171056+0000 mgr.y (mgr.34107) 343 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:22.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:22 smithi089 bash[82331]: cluster 2024-04-13T07:46:21.171056+0000 mgr.y (mgr.34107) 343 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:22.901 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:46:22.902 INFO:teuthology.orchestra.run.smithi089.stderr:dumped all 2024-04-13T07:46:23.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:22 smithi203 bash[68060]: cluster 2024-04-13T07:46:21.171056+0000 mgr.y (mgr.34107) 343 : cluster [DBG] pgmap v280: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:23.586 INFO:teuthology.orchestra.run.smithi089.stdout:{"pg_ready":true,"pg_map":{"version":280,"stamp":"2024-04-13T07:46:21.170703+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790245,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6559,"num_read_kb":13024,"num_write":1146,"num_write_kb":15064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1198,"ondisk_log_size":1198,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":259504,"kb_used_data":24236,"kb_used_omap":41,"kb_used_meta":235094,"kb_avail":749701712,"statfs":{"total":767960285184,"available":767694553088,"internally_reserved":0,"allocated":24817664,"data_stored":19882308,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":42533,"internal_metadata":240736731},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004913"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":77,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.372631+0000","last_change":"2024-04-13T07:44:53.372631+0000","last_active":"2024-04-13T07:44:53.372631+0000","last_peered":"2024-04-13T07:44:53.372631+0000","last_clean":"2024-04-13T07:44:53.372631+0000","last_became_active":"2024-04-13T07:44:53.371713+0000","last_became_peered":"2024-04-13T07:44:53.371713+0000","last_unstale":"2024-04-13T07:44:53.372631+0000","last_undegraded":"2024-04-13T07:44:53.372631+0000","last_fullsized":"2024-04-13T07:44:53.372631+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:10:27.491280+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"70'5","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.349302+0000","last_change":"2024-04-13T07:43:56.606964+0000","last_active":"2024-04-13T07:44:53.349302+0000","last_peered":"2024-04-13T07:44:53.349302+0000","last_clean":"2024-04-13T07:44:53.349302+0000","last_became_active":"2024-04-13T07:43:56.606004+0000","last_became_peered":"2024-04-13T07:43:56.606004+0000","last_unstale":"2024-04-13T07:44:53.349302+0000","last_undegraded":"2024-04-13T07:44:53.349302+0000","last_fullsized":"2024-04-13T07:44:53.349302+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":5,"log_dups_size":0,"ondisk_log_size":5,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:21:34.212932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":582,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":104,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345907+0000","last_change":"2024-04-13T07:44:25.090311+0000","last_active":"2024-04-13T07:44:53.345907+0000","last_peered":"2024-04-13T07:44:53.345907+0000","last_clean":"2024-04-13T07:44:53.345907+0000","last_became_active":"2024-04-13T07:44:25.089751+0000","last_became_peered":"2024-04-13T07:44:25.089751+0000","last_unstale":"2024-04-13T07:44:53.345907+0000","last_undegraded":"2024-04-13T07:44:53.345907+0000","last_fullsized":"2024-04-13T07:44:53.345907+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:54:36.927258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"70'2","reported_seq":93,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.346118+0000","last_change":"2024-04-13T07:44:25.094249+0000","last_active":"2024-04-13T07:44:53.346118+0000","last_peered":"2024-04-13T07:44:53.346118+0000","last_clean":"2024-04-13T07:44:53.346118+0000","last_became_active":"2024-04-13T07:44:25.093610+0000","last_became_peered":"2024-04-13T07:44:25.093610+0000","last_unstale":"2024-04-13T07:44:53.346118+0000","last_undegraded":"2024-04-13T07:44:53.346118+0000","last_fullsized":"2024-04-13T07:44:53.346118+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:10:40.159024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.364318+0000","last_change":"2024-04-13T07:44:53.364318+0000","last_active":"2024-04-13T07:44:53.364318+0000","last_peered":"2024-04-13T07:44:53.364318+0000","last_clean":"2024-04-13T07:44:53.364318+0000","last_became_active":"2024-04-13T07:44:53.363461+0000","last_became_peered":"2024-04-13T07:44:53.363461+0000","last_unstale":"2024-04-13T07:44:53.364318+0000","last_undegraded":"2024-04-13T07:44:53.364318+0000","last_fullsized":"2024-04-13T07:44:53.364318+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:29:49.526269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"69'2","reported_seq":85,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.375884+0000","last_change":"2024-04-13T07:44:53.375884+0000","last_active":"2024-04-13T07:44:53.375884+0000","last_peered":"2024-04-13T07:44:53.375884+0000","last_clean":"2024-04-13T07:44:53.375884+0000","last_became_active":"2024-04-13T07:44:53.374876+0000","last_became_peered":"2024-04-13T07:44:53.374876+0000","last_unstale":"2024-04-13T07:44:53.375884+0000","last_undegraded":"2024-04-13T07:44:53.375884+0000","last_fullsized":"2024-04-13T07:44:53.375884+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:28:37.948151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.383176+0000","last_change":"2024-04-13T07:44:53.383176+0000","last_active":"2024-04-13T07:44:53.383176+0000","last_peered":"2024-04-13T07:44:53.383176+0000","last_clean":"2024-04-13T07:44:53.383176+0000","last_became_active":"2024-04-13T07:44:53.381266+0000","last_became_peered":"2024-04-13T07:44:53.381266+0000","last_unstale":"2024-04-13T07:44:53.383176+0000","last_undegraded":"2024-04-13T07:44:53.383176+0000","last_fullsized":"2024-04-13T07:44:53.383176+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:55:12.355011+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"69'2","reported_seq":100,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.370062+0000","last_change":"2024-04-13T07:44:53.370062+0000","last_active":"2024-04-13T07:44:53.370062+0000","last_peered":"2024-04-13T07:44:53.370062+0000","last_clean":"2024-04-13T07:44:53.370062+0000","last_became_active":"2024-04-13T07:44:53.368994+0000","last_became_peered":"2024-04-13T07:44:53.368994+0000","last_unstale":"2024-04-13T07:44:53.370062+0000","last_undegraded":"2024-04-13T07:44:53.370062+0000","last_fullsized":"2024-04-13T07:44:53.370062+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:26:39.804415+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":104,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.337135+0000","last_change":"2024-04-13T07:44:25.081714+0000","last_active":"2024-04-13T07:44:53.337135+0000","last_peered":"2024-04-13T07:44:53.337135+0000","last_clean":"2024-04-13T07:44:53.337135+0000","last_became_active":"2024-04-13T07:44:25.080865+0000","last_became_peered":"2024-04-13T07:44:25.080865+0000","last_unstale":"2024-04-13T07:44:53.337135+0000","last_undegraded":"2024-04-13T07:44:53.337135+0000","last_fullsized":"2024-04-13T07:44:53.337135+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:21:12.556095+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"70'2","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.372891+0000","last_change":"2024-04-13T07:44:53.372891+0000","last_active":"2024-04-13T07:44:53.372891+0000","last_peered":"2024-04-13T07:44:53.372891+0000","last_clean":"2024-04-13T07:44:53.372891+0000","last_became_active":"2024-04-13T07:44:53.371865+0000","last_became_peered":"2024-04-13T07:44:53.371865+0000","last_unstale":"2024-04-13T07:44:53.372891+0000","last_undegraded":"2024-04-13T07:44:53.372891+0000","last_fullsized":"2024-04-13T07:44:53.372891+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:25:55.170651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":70,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.379949+0000","last_change":"2024-04-13T07:44:53.379949+0000","last_active":"2024-04-13T07:44:53.379949+0000","last_peered":"2024-04-13T07:44:53.379949+0000","last_clean":"2024-04-13T07:44:53.379949+0000","last_became_active":"2024-04-13T07:44:53.378955+0000","last_became_peered":"2024-04-13T07:44:53.378955+0000","last_unstale":"2024-04-13T07:44:53.379949+0000","last_undegraded":"2024-04-13T07:44:53.379949+0000","last_fullsized":"2024-04-13T07:44:53.379949+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:34:19.108316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"70'1","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.374084+0000","last_change":"2024-04-13T07:44:53.373062+0000","last_active":"2024-04-13T07:44:53.374084+0000","last_peered":"2024-04-13T07:44:53.374084+0000","last_clean":"2024-04-13T07:44:53.374084+0000","last_became_active":"2024-04-13T07:44:53.372071+0000","last_became_peered":"2024-04-13T07:44:53.372071+0000","last_unstale":"2024-04-13T07:44:53.374084+0000","last_undegraded":"2024-04-13T07:44:53.374084+0000","last_fullsized":"2024-04-13T07:44:53.374084+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:34:43.826639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"88'83","reported_seq":438,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348667+0000","last_change":"2024-04-13T07:43:56.597620+0000","last_active":"2024-04-13T07:44:53.348667+0000","last_peered":"2024-04-13T07:44:53.348667+0000","last_clean":"2024-04-13T07:44:53.348667+0000","last_became_active":"2024-04-13T07:43:56.596350+0000","last_became_peered":"2024-04-13T07:43:56.596350+0000","last_unstale":"2024-04-13T07:44:53.348667+0000","last_undegraded":"2024-04-13T07:44:53.348667+0000","last_fullsized":"2024-04-13T07:44:53.348667+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":83,"log_dups_size":0,"ondisk_log_size":83,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:01:41.693508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047849900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":578,"num_read_kb":3279,"num_write":60,"num_write_kb":2448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.343890+0000","last_change":"2024-04-13T07:44:25.089344+0000","last_active":"2024-04-13T07:44:53.343890+0000","last_peered":"2024-04-13T07:44:53.343890+0000","last_clean":"2024-04-13T07:44:53.343890+0000","last_became_active":"2024-04-13T07:44:25.088979+0000","last_became_peered":"2024-04-13T07:44:25.088979+0000","last_unstale":"2024-04-13T07:44:53.343890+0000","last_undegraded":"2024-04-13T07:44:53.343890+0000","last_fullsized":"2024-04-13T07:44:53.343890+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:59:10.043540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"88'90","reported_seq":394,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.385448+0000","last_change":"2024-04-13T07:44:53.382271+0000","last_active":"2024-04-13T07:44:53.385448+0000","last_peered":"2024-04-13T07:44:53.385448+0000","last_clean":"2024-04-13T07:44:53.385448+0000","last_became_active":"2024-04-13T07:44:53.379365+0000","last_became_peered":"2024-04-13T07:44:53.379365+0000","last_unstale":"2024-04-13T07:44:53.385448+0000","last_undegraded":"2024-04-13T07:44:53.385448+0000","last_fullsized":"2024-04-13T07:44:53.385448+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":90,"log_dups_size":0,"ondisk_log_size":90,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:57:54.866128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00093873800000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":435,"num_read_kb":340,"num_write":60,"num_write_kb":1053,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"69'2","reported_seq":85,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.346531+0000","last_change":"2024-04-13T07:44:25.092886+0000","last_active":"2024-04-13T07:44:53.346531+0000","last_peered":"2024-04-13T07:44:53.346531+0000","last_clean":"2024-04-13T07:44:53.346531+0000","last_became_active":"2024-04-13T07:44:25.091873+0000","last_became_peered":"2024-04-13T07:44:25.091873+0000","last_unstale":"2024-04-13T07:44:53.346531+0000","last_undegraded":"2024-04-13T07:44:53.346531+0000","last_fullsized":"2024-04-13T07:44:53.346531+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:52:04.470009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.353808+0000","last_change":"2024-04-13T07:41:47.256040+0000","last_active":"2024-04-13T07:44:53.353808+0000","last_peered":"2024-04-13T07:44:53.353808+0000","last_clean":"2024-04-13T07:44:53.353808+0000","last_became_active":"2024-04-13T07:41:47.255668+0000","last_became_peered":"2024-04-13T07:41:47.255668+0000","last_unstale":"2024-04-13T07:44:53.353808+0000","last_undegraded":"2024-04-13T07:44:53.353808+0000","last_fullsized":"2024-04-13T07:44:53.353808+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:18:17.880139+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":62,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.354581+0000","last_change":"2024-04-13T07:43:56.617839+0000","last_active":"2024-04-13T07:44:53.354581+0000","last_peered":"2024-04-13T07:44:53.354581+0000","last_clean":"2024-04-13T07:44:53.354581+0000","last_became_active":"2024-04-13T07:43:56.615696+0000","last_became_peered":"2024-04-13T07:43:56.615696+0000","last_unstale":"2024-04-13T07:44:53.354581+0000","last_undegraded":"2024-04-13T07:44:53.354581+0000","last_fullsized":"2024-04-13T07:44:53.354581+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:19:56.651845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"88'55","reported_seq":364,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.373856+0000","last_change":"2024-04-13T07:44:53.372939+0000","last_active":"2024-04-13T07:44:53.373856+0000","last_peered":"2024-04-13T07:44:53.373856+0000","last_clean":"2024-04-13T07:44:53.373856+0000","last_became_active":"2024-04-13T07:44:53.371733+0000","last_became_peered":"2024-04-13T07:44:53.371733+0000","last_unstale":"2024-04-13T07:44:53.373856+0000","last_undegraded":"2024-04-13T07:44:53.373856+0000","last_fullsized":"2024-04-13T07:44:53.373856+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":55,"log_dups_size":0,"ondisk_log_size":55,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:51:32.511959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0016787200000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":368,"num_read_kb":474,"num_write":36,"num_write_kb":97,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":82,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.352239+0000","last_change":"2024-04-13T07:43:56.601972+0000","last_active":"2024-04-13T07:44:53.352239+0000","last_peered":"2024-04-13T07:44:53.352239+0000","last_clean":"2024-04-13T07:44:53.352239+0000","last_became_active":"2024-04-13T07:43:56.600913+0000","last_became_peered":"2024-04-13T07:43:56.600913+0000","last_unstale":"2024-04-13T07:44:53.352239+0000","last_undegraded":"2024-04-13T07:44:53.352239+0000","last_fullsized":"2024-04-13T07:44:53.352239+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:14:10.946781+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":81,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.371051+0000","last_change":"2024-04-13T07:44:53.370164+0000","last_active":"2024-04-13T07:44:53.371051+0000","last_peered":"2024-04-13T07:44:53.371051+0000","last_clean":"2024-04-13T07:44:53.371051+0000","last_became_active":"2024-04-13T07:44:53.369333+0000","last_became_peered":"2024-04-13T07:44:53.369333+0000","last_unstale":"2024-04-13T07:44:53.371051+0000","last_undegraded":"2024-04-13T07:44:53.371051+0000","last_fullsized":"2024-04-13T07:44:53.371051+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T13:18:23.360881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"88'128","reported_seq":532,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.339728+0000","last_change":"2024-04-13T07:43:31.632113+0000","last_active":"2024-04-13T07:44:52.339728+0000","last_peered":"2024-04-13T07:44:52.339728+0000","last_clean":"2024-04-13T07:44:52.339728+0000","last_became_active":"2024-04-13T07:43:31.630264+0000","last_became_peered":"2024-04-13T07:43:31.630264+0000","last_unstale":"2024-04-13T07:44:52.339728+0000","last_undegraded":"2024-04-13T07:44:52.339728+0000","last_fullsized":"2024-04-13T07:44:52.339728+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":128,"log_dups_size":0,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:20:12.523379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0025084870000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":869,"num_read_kb":1763,"num_write":124,"num_write_kb":1097,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.382584+0000","last_change":"2024-04-13T07:44:53.382584+0000","last_active":"2024-04-13T07:44:53.382584+0000","last_peered":"2024-04-13T07:44:53.382584+0000","last_clean":"2024-04-13T07:44:53.382584+0000","last_became_active":"2024-04-13T07:44:53.381694+0000","last_became_peered":"2024-04-13T07:44:53.381694+0000","last_unstale":"2024-04-13T07:44:53.382584+0000","last_undegraded":"2024-04-13T07:44:53.382584+0000","last_fullsized":"2024-04-13T07:44:53.382584+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:59:30.257522+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.343250+0000","last_change":"2024-04-13T07:44:25.095413+0000","last_active":"2024-04-13T07:44:53.343250+0000","last_peered":"2024-04-13T07:44:53.343250+0000","last_clean":"2024-04-13T07:44:53.343250+0000","last_became_active":"2024-04-13T07:44:25.094703+0000","last_became_peered":"2024-04-13T07:44:25.094703+0000","last_unstale":"2024-04-13T07:44:53.343250+0000","last_undegraded":"2024-04-13T07:44:53.343250+0000","last_fullsized":"2024-04-13T07:44:53.343250+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:30:06.290854+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"92'161","reported_seq":451,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.339221+0000","last_change":"2024-04-13T07:41:47.249040+0000","last_active":"2024-04-13T07:44:53.339221+0000","last_peered":"2024-04-13T07:44:53.339221+0000","last_clean":"2024-04-13T07:44:53.339221+0000","last_became_active":"2024-04-13T07:41:47.246557+0000","last_became_peered":"2024-04-13T07:41:47.246557+0000","last_unstale":"2024-04-13T07:44:53.339221+0000","last_undegraded":"2024-04-13T07:44:53.339221+0000","last_fullsized":"2024-04-13T07:44:53.339221+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":20,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:24:24.657602+0000","last_clean_scrub_stamp":"2024-04-13T07:24:24.657602+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:57:26.656959+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"88'208","reported_seq":647,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.344893+0000","last_change":"2024-04-13T07:43:56.597581+0000","last_active":"2024-04-13T07:44:53.344893+0000","last_peered":"2024-04-13T07:44:53.344893+0000","last_clean":"2024-04-13T07:44:53.344893+0000","last_became_active":"2024-04-13T07:43:56.596278+0000","last_became_peered":"2024-04-13T07:43:56.596278+0000","last_unstale":"2024-04-13T07:44:53.344893+0000","last_undegraded":"2024-04-13T07:44:53.344893+0000","last_fullsized":"2024-04-13T07:44:53.344893+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":208,"log_dups_size":0,"ondisk_log_size":208,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:28:00.889374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001771379,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":818,"num_read_kb":1743,"num_write":191,"num_write_kb":1145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":102,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.353311+0000","last_change":"2024-04-13T07:43:31.646159+0000","last_active":"2024-04-13T07:44:53.353311+0000","last_peered":"2024-04-13T07:44:53.353311+0000","last_clean":"2024-04-13T07:44:53.353311+0000","last_became_active":"2024-04-13T07:43:31.644959+0000","last_became_peered":"2024-04-13T07:43:31.644959+0000","last_unstale":"2024-04-13T07:44:53.353311+0000","last_undegraded":"2024-04-13T07:44:53.353311+0000","last_fullsized":"2024-04-13T07:44:53.353311+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:26:25.210099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":62,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348579+0000","last_change":"2024-04-13T07:43:56.598373+0000","last_active":"2024-04-13T07:44:53.348579+0000","last_peered":"2024-04-13T07:44:53.348579+0000","last_clean":"2024-04-13T07:44:53.348579+0000","last_became_active":"2024-04-13T07:43:56.597433+0000","last_became_peered":"2024-04-13T07:43:56.597433+0000","last_unstale":"2024-04-13T07:44:53.348579+0000","last_undegraded":"2024-04-13T07:44:53.348579+0000","last_fullsized":"2024-04-13T07:44:53.348579+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:22:35.235889+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"88'141","reported_seq":672,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.337098+0000","last_change":"2024-04-13T07:44:25.071644+0000","last_active":"2024-04-13T07:44:53.337098+0000","last_peered":"2024-04-13T07:44:53.337098+0000","last_clean":"2024-04-13T07:44:53.337098+0000","last_became_active":"2024-04-13T07:44:25.069930+0000","last_became_peered":"2024-04-13T07:44:25.069930+0000","last_unstale":"2024-04-13T07:44:53.337098+0000","last_undegraded":"2024-04-13T07:44:53.337098+0000","last_fullsized":"2024-04-13T07:44:53.337098+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":141,"log_dups_size":0,"ondisk_log_size":141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:32:17.505794+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00064946999999999997,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":967,"num_read_kb":766,"num_write":117,"num_write_kb":1090,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.353614+0000","last_change":"2024-04-13T07:44:25.084001+0000","last_active":"2024-04-13T07:44:53.353614+0000","last_peered":"2024-04-13T07:44:53.353614+0000","last_clean":"2024-04-13T07:44:53.353614+0000","last_became_active":"2024-04-13T07:44:25.080976+0000","last_became_peered":"2024-04-13T07:44:25.080976+0000","last_unstale":"2024-04-13T07:44:53.353614+0000","last_undegraded":"2024-04-13T07:44:53.353614+0000","last_fullsized":"2024-04-13T07:44:53.353614+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:46:58.772048+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.379866+0000","last_change":"2024-04-13T07:44:53.379866+0000","last_active":"2024-04-13T07:44:53.379866+0000","last_peered":"2024-04-13T07:44:53.379866+0000","last_clean":"2024-04-13T07:44:53.379866+0000","last_became_active":"2024-04-13T07:44:53.378197+0000","last_became_peered":"2024-04-13T07:44:53.378197+0000","last_unstale":"2024-04-13T07:44:53.379866+0000","last_undegraded":"2024-04-13T07:44:53.379866+0000","last_fullsized":"2024-04-13T07:44:53.379866+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:16:15.536975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"88'69","reported_seq":490,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345319+0000","last_change":"2024-04-13T07:43:31.639857+0000","last_active":"2024-04-13T07:44:53.345319+0000","last_peered":"2024-04-13T07:44:53.345319+0000","last_clean":"2024-04-13T07:44:53.345319+0000","last_became_active":"2024-04-13T07:43:31.638318+0000","last_became_peered":"2024-04-13T07:43:31.638318+0000","last_unstale":"2024-04-13T07:44:53.345319+0000","last_undegraded":"2024-04-13T07:44:53.345319+0000","last_fullsized":"2024-04-13T07:44:53.345319+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":69,"log_dups_size":0,"ondisk_log_size":69,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:09:42.077772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00078461199999999996,"stat_sum":{"num_bytes":4194323,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":480,"num_read_kb":1879,"num_write":69,"num_write_kb":5358,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.386265+0000","last_change":"2024-04-13T07:44:53.381489+0000","last_active":"2024-04-13T07:44:53.386265+0000","last_peered":"2024-04-13T07:44:53.386265+0000","last_clean":"2024-04-13T07:44:53.386265+0000","last_became_active":"2024-04-13T07:44:53.379441+0000","last_became_peered":"2024-04-13T07:44:53.379441+0000","last_unstale":"2024-04-13T07:44:53.386265+0000","last_undegraded":"2024-04-13T07:44:53.386265+0000","last_fullsized":"2024-04-13T07:44:53.386265+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:33:57.409956+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":97,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.363372+0000","last_change":"2024-04-13T07:44:25.083905+0000","last_active":"2024-04-13T07:44:52.363372+0000","last_peered":"2024-04-13T07:44:52.363372+0000","last_clean":"2024-04-13T07:44:52.363372+0000","last_became_active":"2024-04-13T07:44:25.082900+0000","last_became_peered":"2024-04-13T07:44:25.082900+0000","last_unstale":"2024-04-13T07:44:52.363372+0000","last_undegraded":"2024-04-13T07:44:52.363372+0000","last_fullsized":"2024-04-13T07:44:52.363372+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:42:59.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"88'202","reported_seq":938,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.353027+0000","last_change":"2024-04-13T07:43:31.644563+0000","last_active":"2024-04-13T07:44:53.353027+0000","last_peered":"2024-04-13T07:44:53.353027+0000","last_clean":"2024-04-13T07:44:53.353027+0000","last_became_active":"2024-04-13T07:43:31.643395+0000","last_became_peered":"2024-04-13T07:43:31.643395+0000","last_unstale":"2024-04-13T07:44:53.353027+0000","last_undegraded":"2024-04-13T07:44:53.353027+0000","last_fullsized":"2024-04-13T07:44:53.353027+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:30:20.434678+0000","last_clean_scrub_stamp":"2024-04-13T07:30:20.434678+0000","objects_scrubbed":0,"log_size":202,"log_dups_size":0,"ondisk_log_size":202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:57:11.613549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0010273330000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1479,"num_read_kb":1788,"num_write":189,"num_write_kb":330,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":63,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.357356+0000","last_change":"2024-04-13T07:44:53.356007+0000","last_active":"2024-04-13T07:44:53.357356+0000","last_peered":"2024-04-13T07:44:53.357356+0000","last_clean":"2024-04-13T07:44:53.357356+0000","last_became_active":"2024-04-13T07:44:53.355044+0000","last_became_peered":"2024-04-13T07:44:53.355044+0000","last_unstale":"2024-04-13T07:44:53.357356+0000","last_undegraded":"2024-04-13T07:44:53.357356+0000","last_fullsized":"2024-04-13T07:44:53.357356+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:40:11.027657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.344148+0000","last_change":"2024-04-13T07:44:25.096741+0000","last_active":"2024-04-13T07:44:53.344148+0000","last_peered":"2024-04-13T07:44:53.344148+0000","last_clean":"2024-04-13T07:44:53.344148+0000","last_became_active":"2024-04-13T07:44:25.095703+0000","last_became_peered":"2024-04-13T07:44:25.095703+0000","last_unstale":"2024-04-13T07:44:53.344148+0000","last_undegraded":"2024-04-13T07:44:53.344148+0000","last_fullsized":"2024-04-13T07:44:53.344148+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:34:55.859791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.376771+0000","last_change":"2024-04-13T07:44:53.376771+0000","last_active":"2024-04-13T07:44:53.376771+0000","last_peered":"2024-04-13T07:44:53.376771+0000","last_clean":"2024-04-13T07:44:53.376771+0000","last_became_active":"2024-04-13T07:44:53.376030+0000","last_became_peered":"2024-04-13T07:44:53.376030+0000","last_unstale":"2024-04-13T07:44:53.376771+0000","last_undegraded":"2024-04-13T07:44:53.376771+0000","last_fullsized":"2024-04-13T07:44:53.376771+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:10:06.094615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":84,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.370151+0000","last_change":"2024-04-13T07:44:53.370151+0000","last_active":"2024-04-13T07:44:53.370151+0000","last_peered":"2024-04-13T07:44:53.370151+0000","last_clean":"2024-04-13T07:44:53.370151+0000","last_became_active":"2024-04-13T07:44:53.369019+0000","last_became_peered":"2024-04-13T07:44:53.369019+0000","last_unstale":"2024-04-13T07:44:53.370151+0000","last_undegraded":"2024-04-13T07:44:53.370151+0000","last_fullsized":"2024-04-13T07:44:53.370151+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:44:52.317749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.367871+0000","last_change":"2024-04-13T07:44:53.367871+0000","last_active":"2024-04-13T07:44:53.367871+0000","last_peered":"2024-04-13T07:44:53.367871+0000","last_clean":"2024-04-13T07:44:53.367871+0000","last_became_active":"2024-04-13T07:44:53.365445+0000","last_became_peered":"2024-04-13T07:44:53.365445+0000","last_unstale":"2024-04-13T07:44:53.367871+0000","last_undegraded":"2024-04-13T07:44:53.367871+0000","last_fullsized":"2024-04-13T07:44:53.367871+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:55:10.819547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":96,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.382864+0000","last_change":"2024-04-13T07:44:53.381760+0000","last_active":"2024-04-13T07:44:53.382864+0000","last_peered":"2024-04-13T07:44:53.382864+0000","last_clean":"2024-04-13T07:44:53.382864+0000","last_became_active":"2024-04-13T07:44:53.380871+0000","last_became_peered":"2024-04-13T07:44:53.380871+0000","last_unstale":"2024-04-13T07:44:53.382864+0000","last_undegraded":"2024-04-13T07:44:53.382864+0000","last_fullsized":"2024-04-13T07:44:53.382864+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:54:55.694927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":87,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.382451+0000","last_change":"2024-04-13T07:44:53.382451+0000","last_active":"2024-04-13T07:44:53.382451+0000","last_peered":"2024-04-13T07:44:53.382451+0000","last_clean":"2024-04-13T07:44:53.382451+0000","last_became_active":"2024-04-13T07:44:53.381304+0000","last_became_peered":"2024-04-13T07:44:53.381304+0000","last_unstale":"2024-04-13T07:44:53.382451+0000","last_undegraded":"2024-04-13T07:44:53.382451+0000","last_fullsized":"2024-04-13T07:44:53.382451+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:59:02.406363+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"70'3","reported_seq":94,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.361823+0000","last_change":"2024-04-13T07:44:53.361823+0000","last_active":"2024-04-13T07:44:53.361823+0000","last_peered":"2024-04-13T07:44:53.361823+0000","last_clean":"2024-04-13T07:44:53.361823+0000","last_became_active":"2024-04-13T07:44:53.360817+0000","last_became_peered":"2024-04-13T07:44:53.360817+0000","last_unstale":"2024-04-13T07:44:53.361823+0000","last_undegraded":"2024-04-13T07:44:53.361823+0000","last_fullsized":"2024-04-13T07:44:53.361823+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:08:18.067533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":82,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.374091+0000","last_change":"2024-04-13T07:44:53.373053+0000","last_active":"2024-04-13T07:44:53.374091+0000","last_peered":"2024-04-13T07:44:53.374091+0000","last_clean":"2024-04-13T07:44:53.374091+0000","last_became_active":"2024-04-13T07:44:53.372124+0000","last_became_peered":"2024-04-13T07:44:53.372124+0000","last_unstale":"2024-04-13T07:44:53.374091+0000","last_undegraded":"2024-04-13T07:44:53.374091+0000","last_fullsized":"2024-04-13T07:44:53.374091+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:54:02.008646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":82,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348275+0000","last_change":"2024-04-13T07:44:25.089560+0000","last_active":"2024-04-13T07:44:53.348275+0000","last_peered":"2024-04-13T07:44:53.348275+0000","last_clean":"2024-04-13T07:44:53.348275+0000","last_became_active":"2024-04-13T07:44:25.088757+0000","last_became_peered":"2024-04-13T07:44:25.088757+0000","last_unstale":"2024-04-13T07:44:53.348275+0000","last_undegraded":"2024-04-13T07:44:53.348275+0000","last_fullsized":"2024-04-13T07:44:53.348275+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:03:49.823397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348200+0000","last_change":"2024-04-13T07:43:31.643631+0000","last_active":"2024-04-13T07:44:53.348200+0000","last_peered":"2024-04-13T07:44:53.348200+0000","last_clean":"2024-04-13T07:44:53.348200+0000","last_became_active":"2024-04-13T07:43:31.641747+0000","last_became_peered":"2024-04-13T07:43:31.641747+0000","last_unstale":"2024-04-13T07:44:53.348200+0000","last_undegraded":"2024-04-13T07:44:53.348200+0000","last_fullsized":"2024-04-13T07:44:53.348200+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:21:55.189042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"69'1","reported_seq":68,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345509+0000","last_change":"2024-04-13T07:32:03.078696+0000","last_active":"2024-04-13T07:44:53.345509+0000","last_peered":"2024-04-13T07:44:53.345509+0000","last_clean":"2024-04-13T07:44:53.345509+0000","last_became_active":"2024-04-13T07:32:03.077798+0000","last_became_peered":"2024-04-13T07:32:03.077798+0000","last_unstale":"2024-04-13T07:44:53.345509+0000","last_undegraded":"2024-04-13T07:44:53.345509+0000","last_fullsized":"2024-04-13T07:44:53.345509+0000","mapping_epoch":65,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:32:02.676056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":102,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.384421+0000","last_change":"2024-04-13T07:44:53.384421+0000","last_active":"2024-04-13T07:44:53.384421+0000","last_peered":"2024-04-13T07:44:53.384421+0000","last_clean":"2024-04-13T07:44:53.384421+0000","last_became_active":"2024-04-13T07:44:53.383605+0000","last_became_peered":"2024-04-13T07:44:53.383605+0000","last_unstale":"2024-04-13T07:44:53.384421+0000","last_undegraded":"2024-04-13T07:44:53.384421+0000","last_fullsized":"2024-04-13T07:44:53.384421+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:31:47.009049+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"69'2","reported_seq":100,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.364990+0000","last_change":"2024-04-13T07:44:53.364027+0000","last_active":"2024-04-13T07:44:53.364990+0000","last_peered":"2024-04-13T07:44:53.364990+0000","last_clean":"2024-04-13T07:44:53.364990+0000","last_became_active":"2024-04-13T07:44:53.362973+0000","last_became_peered":"2024-04-13T07:44:53.362973+0000","last_unstale":"2024-04-13T07:44:53.364990+0000","last_undegraded":"2024-04-13T07:44:53.364990+0000","last_fullsized":"2024-04-13T07:44:53.364990+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:54:20.558297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":6,"num_read_kb":6,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":66,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.346597+0000","last_change":"2024-04-13T07:32:03.141831+0000","last_active":"2024-04-13T07:44:53.346597+0000","last_peered":"2024-04-13T07:44:53.346597+0000","last_clean":"2024-04-13T07:44:53.346597+0000","last_became_active":"2024-04-13T07:32:03.141053+0000","last_became_peered":"2024-04-13T07:32:03.141053+0000","last_unstale":"2024-04-13T07:44:53.346597+0000","last_undegraded":"2024-04-13T07:44:53.346597+0000","last_fullsized":"2024-04-13T07:44:53.346597+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:24:08.376151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"69'1","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.365521+0000","last_change":"2024-04-13T07:44:53.364935+0000","last_active":"2024-04-13T07:44:53.365521+0000","last_peered":"2024-04-13T07:44:53.365521+0000","last_clean":"2024-04-13T07:44:53.365521+0000","last_became_active":"2024-04-13T07:44:53.364004+0000","last_became_peered":"2024-04-13T07:44:53.364004+0000","last_unstale":"2024-04-13T07:44:53.365521+0000","last_undegraded":"2024-04-13T07:44:53.365521+0000","last_fullsized":"2024-04-13T07:44:53.365521+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:07:08.723096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":47,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.346235+0000","last_change":"2024-04-13T07:44:25.083540+0000","last_active":"2024-04-13T07:44:53.346235+0000","last_peered":"2024-04-13T07:44:53.346235+0000","last_clean":"2024-04-13T07:44:53.346235+0000","last_became_active":"2024-04-13T07:44:25.079738+0000","last_became_peered":"2024-04-13T07:44:25.079738+0000","last_unstale":"2024-04-13T07:44:53.346235+0000","last_undegraded":"2024-04-13T07:44:53.346235+0000","last_fullsized":"2024-04-13T07:44:53.346235+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:52:46.929779+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,5],"acting":[3,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"69'8","reported_seq":97,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348493+0000","last_change":"2024-04-13T07:41:47.264342+0000","last_active":"2024-04-13T07:44:53.348493+0000","last_peered":"2024-04-13T07:44:53.348493+0000","last_clean":"2024-04-13T07:44:53.348493+0000","last_became_active":"2024-04-13T07:41:47.262850+0000","last_became_peered":"2024-04-13T07:41:47.262850+0000","last_unstale":"2024-04-13T07:44:53.348493+0000","last_undegraded":"2024-04-13T07:44:53.348493+0000","last_fullsized":"2024-04-13T07:44:53.348493+0000","mapping_epoch":96,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":97,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:12:51.340350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":24,"num_read_kb":24,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.350362+0000","last_change":"2024-04-13T07:44:25.085323+0000","last_active":"2024-04-13T07:44:53.350362+0000","last_peered":"2024-04-13T07:44:53.350362+0000","last_clean":"2024-04-13T07:44:53.350362+0000","last_became_active":"2024-04-13T07:44:25.084440+0000","last_became_peered":"2024-04-13T07:44:25.084440+0000","last_unstale":"2024-04-13T07:44:53.350362+0000","last_undegraded":"2024-04-13T07:44:53.350362+0000","last_fullsized":"2024-04-13T07:44:53.350362+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:53:26.066053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":81,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.362437+0000","last_change":"2024-04-13T07:44:53.361144+0000","last_active":"2024-04-13T07:44:53.362437+0000","last_peered":"2024-04-13T07:44:53.362437+0000","last_clean":"2024-04-13T07:44:53.362437+0000","last_became_active":"2024-04-13T07:44:53.360244+0000","last_became_peered":"2024-04-13T07:44:53.360244+0000","last_unstale":"2024-04-13T07:44:53.362437+0000","last_undegraded":"2024-04-13T07:44:53.362437+0000","last_fullsized":"2024-04-13T07:44:53.362437+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T12:39:43.030717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":80,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.363137+0000","last_change":"2024-04-13T07:44:25.083258+0000","last_active":"2024-04-13T07:44:52.363137+0000","last_peered":"2024-04-13T07:44:52.363137+0000","last_clean":"2024-04-13T07:44:52.363137+0000","last_became_active":"2024-04-13T07:44:25.082364+0000","last_became_peered":"2024-04-13T07:44:25.082364+0000","last_unstale":"2024-04-13T07:44:52.363137+0000","last_undegraded":"2024-04-13T07:44:52.363137+0000","last_fullsized":"2024-04-13T07:44:52.363137+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T19:24:05.070684+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.348766+0000","last_change":"2024-04-13T07:44:25.081436+0000","last_active":"2024-04-13T07:44:53.348766+0000","last_peered":"2024-04-13T07:44:53.348766+0000","last_clean":"2024-04-13T07:44:53.348766+0000","last_became_active":"2024-04-13T07:44:25.080738+0000","last_became_peered":"2024-04-13T07:44:25.080738+0000","last_unstale":"2024-04-13T07:44:53.348766+0000","last_undegraded":"2024-04-13T07:44:53.348766+0000","last_fullsized":"2024-04-13T07:44:53.348766+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T14:01:13.465690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345736+0000","last_change":"2024-04-13T07:44:25.097395+0000","last_active":"2024-04-13T07:44:53.345736+0000","last_peered":"2024-04-13T07:44:53.345736+0000","last_clean":"2024-04-13T07:44:53.345736+0000","last_became_active":"2024-04-13T07:44:25.096490+0000","last_became_peered":"2024-04-13T07:44:25.096490+0000","last_unstale":"2024-04-13T07:44:53.345736+0000","last_undegraded":"2024-04-13T07:44:53.345736+0000","last_fullsized":"2024-04-13T07:44:53.345736+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:40:03.850170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"69'2","reported_seq":100,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.361793+0000","last_change":"2024-04-13T07:44:53.360987+0000","last_active":"2024-04-13T07:44:53.361793+0000","last_peered":"2024-04-13T07:44:53.361793+0000","last_clean":"2024-04-13T07:44:53.361793+0000","last_became_active":"2024-04-13T07:44:53.359750+0000","last_became_peered":"2024-04-13T07:44:53.359750+0000","last_unstale":"2024-04-13T07:44:53.361793+0000","last_undegraded":"2024-04-13T07:44:53.361793+0000","last_fullsized":"2024-04-13T07:44:53.361793+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:28:14.428704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":4,"num_read_kb":8,"num_write":3,"num_write_kb":7,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":77,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.377475+0000","last_change":"2024-04-13T07:44:53.377475+0000","last_active":"2024-04-13T07:44:53.377475+0000","last_peered":"2024-04-13T07:44:53.377475+0000","last_clean":"2024-04-13T07:44:53.377475+0000","last_became_active":"2024-04-13T07:44:53.374822+0000","last_became_peered":"2024-04-13T07:44:53.374822+0000","last_unstale":"2024-04-13T07:44:53.377475+0000","last_undegraded":"2024-04-13T07:44:53.377475+0000","last_fullsized":"2024-04-13T07:44:53.377475+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:43:35.887419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"92'23","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.379615+0000","last_change":"2024-04-13T07:44:53.379615+0000","last_active":"2024-04-13T07:44:53.379615+0000","last_peered":"2024-04-13T07:44:53.379615+0000","last_clean":"2024-04-13T07:44:53.379615+0000","last_became_active":"2024-04-13T07:44:53.378060+0000","last_became_peered":"2024-04-13T07:44:53.378060+0000","last_unstale":"2024-04-13T07:44:53.379615+0000","last_undegraded":"2024-04-13T07:44:53.379615+0000","last_fullsized":"2024-04-13T07:44:53.379615+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":23,"log_dups_size":0,"ondisk_log_size":23,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:59:57.314373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4756,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":10,"num_read_kb":9,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.352186+0000","last_change":"2024-04-13T07:43:56.598624+0000","last_active":"2024-04-13T07:44:53.352186+0000","last_peered":"2024-04-13T07:44:53.352186+0000","last_clean":"2024-04-13T07:44:53.352186+0000","last_became_active":"2024-04-13T07:43:56.597710+0000","last_became_peered":"2024-04-13T07:43:56.597710+0000","last_unstale":"2024-04-13T07:44:53.352186+0000","last_undegraded":"2024-04-13T07:44:53.352186+0000","last_fullsized":"2024-04-13T07:44:53.352186+0000","mapping_epoch":106,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":107,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T07:47:49.807918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"69'3","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.370798+0000","last_change":"2024-04-13T07:44:53.370798+0000","last_active":"2024-04-13T07:44:53.370798+0000","last_peered":"2024-04-13T07:44:53.370798+0000","last_clean":"2024-04-13T07:44:53.370798+0000","last_became_active":"2024-04-13T07:44:53.369966+0000","last_became_peered":"2024-04-13T07:44:53.369966+0000","last_unstale":"2024-04-13T07:44:53.370798+0000","last_undegraded":"2024-04-13T07:44:53.370798+0000","last_fullsized":"2024-04-13T07:44:53.370798+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:02:25.087869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":7,"num_read_kb":7,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":33,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.345607+0000","last_change":"2024-04-13T07:44:25.094541+0000","last_active":"2024-04-13T07:44:53.345607+0000","last_peered":"2024-04-13T07:44:53.345607+0000","last_clean":"2024-04-13T07:44:53.345607+0000","last_became_active":"2024-04-13T07:44:25.093753+0000","last_became_peered":"2024-04-13T07:44:25.093753+0000","last_unstale":"2024-04-13T07:44:53.345607+0000","last_undegraded":"2024-04-13T07:44:53.345607+0000","last_fullsized":"2024-04-13T07:44:53.345607+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T10:19:45.907198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":81,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.360540+0000","last_change":"2024-04-13T07:43:31.636805+0000","last_active":"2024-04-13T07:44:52.360540+0000","last_peered":"2024-04-13T07:44:52.360540+0000","last_clean":"2024-04-13T07:44:52.360540+0000","last_became_active":"2024-04-13T07:43:31.635810+0000","last_became_peered":"2024-04-13T07:43:31.635810+0000","last_unstale":"2024-04-13T07:44:52.360540+0000","last_undegraded":"2024-04-13T07:44:52.360540+0000","last_fullsized":"2024-04-13T07:44:52.360540+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T18:36:07.875306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.343004+0000","last_change":"2024-04-13T07:44:25.097296+0000","last_active":"2024-04-13T07:44:53.343004+0000","last_peered":"2024-04-13T07:44:53.343004+0000","last_clean":"2024-04-13T07:44:53.343004+0000","last_became_active":"2024-04-13T07:44:25.096518+0000","last_became_peered":"2024-04-13T07:44:25.096518+0000","last_unstale":"2024-04-13T07:44:53.343004+0000","last_undegraded":"2024-04-13T07:44:53.343004+0000","last_fullsized":"2024-04-13T07:44:53.343004+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:35:13.762142+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":72,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.349457+0000","last_change":"2024-04-13T07:44:25.083097+0000","last_active":"2024-04-13T07:44:53.349457+0000","last_peered":"2024-04-13T07:44:53.349457+0000","last_clean":"2024-04-13T07:44:53.349457+0000","last_became_active":"2024-04-13T07:44:25.082218+0000","last_became_peered":"2024-04-13T07:44:25.082218+0000","last_unstale":"2024-04-13T07:44:53.349457+0000","last_undegraded":"2024-04-13T07:44:53.349457+0000","last_fullsized":"2024-04-13T07:44:53.349457+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T16:23:08.217840+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":80,"reported_epoch":116,"state":"active+clean","last_fresh":"2024-04-13T07:44:52.356724+0000","last_change":"2024-04-13T07:43:31.635070+0000","last_active":"2024-04-13T07:44:52.356724+0000","last_peered":"2024-04-13T07:44:52.356724+0000","last_clean":"2024-04-13T07:44:52.356724+0000","last_became_active":"2024-04-13T07:43:31.634074+0000","last_became_peered":"2024-04-13T07:43:31.634074+0000","last_unstale":"2024-04-13T07:44:52.356724+0000","last_undegraded":"2024-04-13T07:44:52.356724+0000","last_fullsized":"2024-04-13T07:44:52.356724+0000","mapping_epoch":101,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":102,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T08:16:42.369895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.343599+0000","last_change":"2024-04-13T07:44:25.089335+0000","last_active":"2024-04-13T07:44:53.343599+0000","last_peered":"2024-04-13T07:44:53.343599+0000","last_clean":"2024-04-13T07:44:53.343599+0000","last_became_active":"2024-04-13T07:44:25.088925+0000","last_became_peered":"2024-04-13T07:44:25.088925+0000","last_unstale":"2024-04-13T07:44:53.343599+0000","last_undegraded":"2024-04-13T07:44:53.343599+0000","last_fullsized":"2024-04-13T07:44:53.343599+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T17:42:45.826951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":73,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.350745+0000","last_change":"2024-04-13T07:44:25.086154+0000","last_active":"2024-04-13T07:44:53.350745+0000","last_peered":"2024-04-13T07:44:53.350745+0000","last_clean":"2024-04-13T07:44:53.350745+0000","last_became_active":"2024-04-13T07:44:25.085727+0000","last_became_peered":"2024-04-13T07:44:25.085727+0000","last_unstale":"2024-04-13T07:44:53.350745+0000","last_undegraded":"2024-04-13T07:44:53.350745+0000","last_fullsized":"2024-04-13T07:44:53.350745+0000","mapping_epoch":111,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":112,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T09:20:24.586059+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":71,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.385363+0000","last_change":"2024-04-13T07:44:53.381154+0000","last_active":"2024-04-13T07:44:53.385363+0000","last_peered":"2024-04-13T07:44:53.385363+0000","last_clean":"2024-04-13T07:44:53.385363+0000","last_became_active":"2024-04-13T07:44:53.379300+0000","last_became_peered":"2024-04-13T07:44:53.379300+0000","last_unstale":"2024-04-13T07:44:53.385363+0000","last_undegraded":"2024-04-13T07:44:53.385363+0000","last_fullsized":"2024-04-13T07:44:53.385363+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T15:22:35.755043+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":107,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.380849+0000","last_change":"2024-04-13T07:44:53.380849+0000","last_active":"2024-04-13T07:44:53.380849+0000","last_peered":"2024-04-13T07:44:53.380849+0000","last_clean":"2024-04-13T07:44:53.380849+0000","last_became_active":"2024-04-13T07:44:53.380000+0000","last_became_peered":"2024-04-13T07:44:53.380000+0000","last_unstale":"2024-04-13T07:44:53.380849+0000","last_undegraded":"2024-04-13T07:44:53.380849+0000","last_fullsized":"2024-04-13T07:44:53.380849+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:03.016070+0000","last_clean_scrub_stamp":"2024-04-13T07:32:03.016070+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:19:53.653413+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"70'2","reported_seq":100,"reported_epoch":117,"state":"active+clean","last_fresh":"2024-04-13T07:44:53.368700+0000","last_change":"2024-04-13T07:44:53.368700+0000","last_active":"2024-04-13T07:44:53.368700+0000","last_peered":"2024-04-13T07:44:53.368700+0000","last_clean":"2024-04-13T07:44:53.368700+0000","last_became_active":"2024-04-13T07:44:53.365796+0000","last_became_peered":"2024-04-13T07:44:53.365796+0000","last_unstale":"2024-04-13T07:44:53.368700+0000","last_undegraded":"2024-04-13T07:44:53.368700+0000","last_fullsized":"2024-04-13T07:44:53.368700+0000","mapping_epoch":116,"log_start":"0'0","ondisk_log_start":"0'0","created":65,"last_epoch_clean":117,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2024-04-13T07:32:02.004771+0000","last_clean_scrub_stamp":"2024-04-13T07:32:02.004771+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2024-04-14T11:35:41.555072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5554,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":65,"num_read_kb":68,"num_write":61,"num_write_kb":30,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":24245,"internal_metadata":0},"log_size":61,"ondisk_log_size":61,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12032,"num_write":846,"num_write_kb":12618,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5545,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":55,"seq":236223201495,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37084,"kb_used_data":1872,"kb_used_omap":1,"kb_used_meta":35198,"kb_avail":93708068,"statfs":{"total":95995035648,"available":95957061632,"internally_reserved":0,"allocated":1916928,"data_stored":1284396,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1589,"internal_metadata":36043211},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":6,"up_from":47,"seq":201863463137,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32972,"kb_used_data":5952,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93712180,"statfs":{"total":95995035648,"available":95961272320,"internally_reserved":0,"allocated":6094848,"data_stored":5477969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1593,"internal_metadata":27654599},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":1,"up_from":101,"seq":433791696932,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":37056,"kb_used_data":5364,"kb_used_omap":7,"kb_used_meta":31672,"kb_avail":93708096,"statfs":{"total":95995035648,"available":95957090304,"internally_reserved":0,"allocated":5492736,"data_stored":4883391,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7566,"internal_metadata":32432754},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":0,"up_from":96,"seq":412316860473,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":29392,"kb_used_data":1852,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93715760,"statfs":{"total":95995035648,"available":95964938240,"internally_reserved":0,"allocated":1896448,"data_stored":1280040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8046,"internal_metadata":28172434},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":2,"up_from":106,"seq":455266533407,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28812,"kb_used_data":1272,"kb_used_omap":8,"kb_used_meta":27511,"kb_avail":93716340,"statfs":{"total":95995035648,"available":95965532160,"internally_reserved":0,"allocated":1302528,"data_stored":689684,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":8387,"internal_metadata":28172093},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":3,"up_from":111,"seq":476741369881,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32968,"kb_used_data":1268,"kb_used_omap":6,"kb_used_meta":31673,"kb_avail":93712184,"statfs":{"total":95995035648,"available":95961276416,"internally_reserved":0,"allocated":1298432,"data_stored":689158,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6212,"internal_metadata":32434108},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":4,"up_from":116,"seq":498216206355,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28828,"kb_used_data":1288,"kb_used_omap":7,"kb_used_meta":27512,"kb_avail":93716324,"statfs":{"total":95995035648,"available":95965515776,"internally_reserved":0,"allocated":1318912,"data_stored":693970,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7546,"internal_metadata":28172934},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]},{"osd":5,"up_from":39,"seq":167503724780,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32392,"kb_used_data":5368,"kb_used_omap":1,"kb_used_meta":27006,"kb_avail":93712760,"statfs":{"total":95995035648,"available":95961866240,"internally_reserved":0,"allocated":5496832,"data_stored":4883700,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1594,"internal_metadata":27654598},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4242,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":1303,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":604,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5972,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2552,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3317,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":5952,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":652,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4972,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2024-04-13T07:46:23.589 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2024-04-13T07:46:23.589 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2024-04-13T07:46:23.590 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2024-04-13T07:46:23.590 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:46:25.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:24 smithi203 bash[68060]: audit 2024-04-13T07:46:22.894372+0000 mgr.y (mgr.34107) 344 : audit [DBG] from='client.44468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:46:25.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:24 smithi203 bash[68060]: cluster 2024-04-13T07:46:23.171940+0000 mgr.y (mgr.34107) 345 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:24 smithi089 bash[82331]: audit 2024-04-13T07:46:22.894372+0000 mgr.y (mgr.34107) 344 : audit [DBG] from='client.44468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:46:25.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:24 smithi089 bash[82331]: cluster 2024-04-13T07:46:23.171940+0000 mgr.y (mgr.34107) 345 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:25.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:24 smithi089 bash[84777]: audit 2024-04-13T07:46:22.894372+0000 mgr.y (mgr.34107) 344 : audit [DBG] from='client.44468 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2024-04-13T07:46:25.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:24 smithi089 bash[84777]: cluster 2024-04-13T07:46:23.171940+0000 mgr.y (mgr.34107) 345 : cluster [DBG] pgmap v281: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:26.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:46:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:46:25] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:46:27.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:26 smithi203 bash[68060]: cluster 2024-04-13T07:46:25.172684+0000 mgr.y (mgr.34107) 346 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:27.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:26 smithi089 bash[84777]: cluster 2024-04-13T07:46:25.172684+0000 mgr.y (mgr.34107) 346 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:27.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:26 smithi089 bash[82331]: cluster 2024-04-13T07:46:25.172684+0000 mgr.y (mgr.34107) 346 : cluster [DBG] pgmap v282: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:28.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:27 smithi203 bash[68060]: audit 2024-04-13T07:46:27.076909+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:28.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:27 smithi089 bash[82331]: audit 2024-04-13T07:46:27.076909+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:28.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:27 smithi089 bash[84777]: audit 2024-04-13T07:46:27.076909+0000 mon.a (mon.0) 614 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:28.329 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:46:29.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:28 smithi203 bash[68060]: cluster 2024-04-13T07:46:27.173457+0000 mgr.y (mgr.34107) 347 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:29.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:28 smithi089 bash[82331]: cluster 2024-04-13T07:46:27.173457+0000 mgr.y (mgr.34107) 347 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:29.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:28 smithi089 bash[84777]: cluster 2024-04-13T07:46:27.173457+0000 mgr.y (mgr.34107) 347 : cluster [DBG] pgmap v283: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:30 smithi089 bash[82331]: cluster 2024-04-13T07:46:29.173969+0000 mgr.y (mgr.34107) 348 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:30.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:30 smithi089 bash[84777]: cluster 2024-04-13T07:46:29.173969+0000 mgr.y (mgr.34107) 348 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:30.966 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:46:30.967 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:46:31.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:30 smithi203 bash[68060]: cluster 2024-04-13T07:46:29.173969+0000 mgr.y (mgr.34107) 348 : cluster [DBG] pgmap v284: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:32.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:31 smithi203 bash[68060]: audit 2024-04-13T07:46:30.972351+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.89:0/3163846253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:32.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:31 smithi089 bash[82331]: audit 2024-04-13T07:46:30.972351+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.89:0/3163846253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:32.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:31 smithi089 bash[84777]: audit 2024-04-13T07:46:30.972351+0000 mon.a (mon.0) 615 : audit [DBG] from='client.? 172.21.15.89:0/3163846253' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:32.761 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:46:32 smithi203 bash[60252]: ts=2024-04-13T07:46:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:46:33.016 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:32 smithi203 bash[68060]: cluster 2024-04-13T07:46:31.174695+0000 mgr.y (mgr.34107) 349 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:33.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:32 smithi089 bash[84777]: cluster 2024-04-13T07:46:31.174695+0000 mgr.y (mgr.34107) 349 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:33.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:32 smithi089 bash[82331]: cluster 2024-04-13T07:46:31.174695+0000 mgr.y (mgr.34107) 349 : cluster [DBG] pgmap v285: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:34.615 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:46:35.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:34 smithi203 bash[68060]: cluster 2024-04-13T07:46:33.175632+0000 mgr.y (mgr.34107) 350 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:35.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:34 smithi089 bash[84777]: cluster 2024-04-13T07:46:33.175632+0000 mgr.y (mgr.34107) 350 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:35.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:34 smithi089 bash[82331]: cluster 2024-04-13T07:46:33.175632+0000 mgr.y (mgr.34107) 350 : cluster [DBG] pgmap v286: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:36.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:46:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:46:35] "GET /metrics HTTP/1.1" 200 34076 "" "Prometheus/2.43.0" 2024-04-13T07:46:37.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:36 smithi089 bash[84777]: cluster 2024-04-13T07:46:35.176334+0000 mgr.y (mgr.34107) 351 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:37.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:36 smithi089 bash[82331]: cluster 2024-04-13T07:46:35.176334+0000 mgr.y (mgr.34107) 351 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:37.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:36 smithi203 bash[68060]: cluster 2024-04-13T07:46:35.176334+0000 mgr.y (mgr.34107) 351 : cluster [DBG] pgmap v287: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:39.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:38 smithi089 bash[84777]: cluster 2024-04-13T07:46:37.177127+0000 mgr.y (mgr.34107) 352 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:39.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:38 smithi089 bash[82331]: cluster 2024-04-13T07:46:37.177127+0000 mgr.y (mgr.34107) 352 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:39.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:38 smithi203 bash[68060]: cluster 2024-04-13T07:46:37.177127+0000 mgr.y (mgr.34107) 352 : cluster [DBG] pgmap v288: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:39.365 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:46:41.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:40 smithi089 bash[82331]: cluster 2024-04-13T07:46:39.177994+0000 mgr.y (mgr.34107) 353 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:41.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:40 smithi089 bash[84777]: cluster 2024-04-13T07:46:39.177994+0000 mgr.y (mgr.34107) 353 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:41.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:40 smithi203 bash[68060]: cluster 2024-04-13T07:46:39.177994+0000 mgr.y (mgr.34107) 353 : cluster [DBG] pgmap v289: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:42.192 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:46:42.192 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:46:42.767 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:46:42 smithi203 bash[60252]: ts=2024-04-13T07:46:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:46:43.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:42 smithi089 bash[82331]: cluster 2024-04-13T07:46:41.178775+0000 mgr.y (mgr.34107) 354 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:43.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:42 smithi089 bash[82331]: audit 2024-04-13T07:46:42.077355+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:43.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:42 smithi089 bash[82331]: audit 2024-04-13T07:46:42.197545+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.89:0/670047759' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:43.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:42 smithi089 bash[84777]: cluster 2024-04-13T07:46:41.178775+0000 mgr.y (mgr.34107) 354 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:43.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:42 smithi089 bash[84777]: audit 2024-04-13T07:46:42.077355+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:43.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:42 smithi089 bash[84777]: audit 2024-04-13T07:46:42.197545+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.89:0/670047759' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:43.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:42 smithi203 bash[68060]: cluster 2024-04-13T07:46:41.178775+0000 mgr.y (mgr.34107) 354 : cluster [DBG] pgmap v290: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:43.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:42 smithi203 bash[68060]: audit 2024-04-13T07:46:42.077355+0000 mon.a (mon.0) 616 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:43.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:42 smithi203 bash[68060]: audit 2024-04-13T07:46:42.197545+0000 mon.a (mon.0) 617 : audit [DBG] from='client.? 172.21.15.89:0/670047759' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:45.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:44 smithi089 bash[84777]: cluster 2024-04-13T07:46:43.179708+0000 mgr.y (mgr.34107) 355 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:45.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:44 smithi089 bash[82331]: cluster 2024-04-13T07:46:43.179708+0000 mgr.y (mgr.34107) 355 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:45.266 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:44 smithi203 bash[68060]: cluster 2024-04-13T07:46:43.179708+0000 mgr.y (mgr.34107) 355 : cluster [DBG] pgmap v291: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:45.778 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:46:46.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:45 smithi089 bash[82331]: cluster 2024-04-13T07:46:45.180426+0000 mgr.y (mgr.34107) 356 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:46.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:46:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:46:45] "GET /metrics HTTP/1.1" 200 34077 "" "Prometheus/2.43.0" 2024-04-13T07:46:46.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:45 smithi089 bash[84777]: cluster 2024-04-13T07:46:45.180426+0000 mgr.y (mgr.34107) 356 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:46.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:45 smithi203 bash[68060]: cluster 2024-04-13T07:46:45.180426+0000 mgr.y (mgr.34107) 356 : cluster [DBG] pgmap v292: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:48.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:48 smithi203 bash[68060]: cluster 2024-04-13T07:46:47.181216+0000 mgr.y (mgr.34107) 357 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:48.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:48 smithi089 bash[82331]: cluster 2024-04-13T07:46:47.181216+0000 mgr.y (mgr.34107) 357 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:48.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:48 smithi089 bash[84777]: cluster 2024-04-13T07:46:47.181216+0000 mgr.y (mgr.34107) 357 : cluster [DBG] pgmap v293: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:50.516 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:50 smithi203 bash[68060]: cluster 2024-04-13T07:46:49.182083+0000 mgr.y (mgr.34107) 358 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:50.518 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:46:50.541 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:50 smithi089 bash[84777]: cluster 2024-04-13T07:46:49.182083+0000 mgr.y (mgr.34107) 358 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:50.541 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:50 smithi089 bash[82331]: cluster 2024-04-13T07:46:49.182083+0000 mgr.y (mgr.34107) 358 : cluster [DBG] pgmap v294: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:52.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:52 smithi203 bash[68060]: cluster 2024-04-13T07:46:51.182936+0000 mgr.y (mgr.34107) 359 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:52.517 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:46:52 smithi203 bash[60252]: ts=2024-04-13T07:46:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:46:52.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:52 smithi089 bash[84777]: cluster 2024-04-13T07:46:51.182936+0000 mgr.y (mgr.34107) 359 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:52.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:52 smithi089 bash[82331]: cluster 2024-04-13T07:46:51.182936+0000 mgr.y (mgr.34107) 359 : cluster [DBG] pgmap v295: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:53.327 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:46:53.327 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:46:54.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:54 smithi203 bash[68060]: cluster 2024-04-13T07:46:53.183832+0000 mgr.y (mgr.34107) 360 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:54.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:54 smithi203 bash[68060]: audit 2024-04-13T07:46:53.332908+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.89:0/2472994404' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:54.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:54 smithi089 bash[82331]: cluster 2024-04-13T07:46:53.183832+0000 mgr.y (mgr.34107) 360 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:54.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:54 smithi089 bash[82331]: audit 2024-04-13T07:46:53.332908+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.89:0/2472994404' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:54.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:54 smithi089 bash[84777]: cluster 2024-04-13T07:46:53.183832+0000 mgr.y (mgr.34107) 360 : cluster [DBG] pgmap v296: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:54.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:54 smithi089 bash[84777]: audit 2024-04-13T07:46:53.332908+0000 mon.a (mon.0) 618 : audit [DBG] from='client.? 172.21.15.89:0/2472994404' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:46:56.248 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:46:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:46:55] "GET /metrics HTTP/1.1" 200 34077 "" "Prometheus/2.43.0" 2024-04-13T07:46:56.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:56 smithi203 bash[68060]: cluster 2024-04-13T07:46:55.184551+0000 mgr.y (mgr.34107) 361 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:56.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:56 smithi089 bash[84777]: cluster 2024-04-13T07:46:55.184551+0000 mgr.y (mgr.34107) 361 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:56.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:56 smithi089 bash[82331]: cluster 2024-04-13T07:46:55.184551+0000 mgr.y (mgr.34107) 361 : cluster [DBG] pgmap v297: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:56.941 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:46:57.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:57 smithi203 bash[68060]: audit 2024-04-13T07:46:57.077997+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:57.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:57 smithi089 bash[82331]: audit 2024-04-13T07:46:57.077997+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:57.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:57 smithi089 bash[84777]: audit 2024-04-13T07:46:57.077997+0000 mon.a (mon.0) 619 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:46:58.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:46:58 smithi203 bash[68060]: cluster 2024-04-13T07:46:57.185033+0000 mgr.y (mgr.34107) 362 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:58.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:46:58 smithi089 bash[84777]: cluster 2024-04-13T07:46:57.185033+0000 mgr.y (mgr.34107) 362 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:46:58.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:46:58 smithi089 bash[82331]: cluster 2024-04-13T07:46:57.185033+0000 mgr.y (mgr.34107) 362 : cluster [DBG] pgmap v298: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:00.517 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:00 smithi203 bash[68060]: cluster 2024-04-13T07:46:59.185845+0000 mgr.y (mgr.34107) 363 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:00.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:00 smithi089 bash[84777]: cluster 2024-04-13T07:46:59.185845+0000 mgr.y (mgr.34107) 363 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:00.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:00 smithi089 bash[82331]: cluster 2024-04-13T07:46:59.185845+0000 mgr.y (mgr.34107) 363 : cluster [DBG] pgmap v299: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:01.728 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:47:02.519 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:02 smithi089 bash[82331]: cluster 2024-04-13T07:47:01.186661+0000 mgr.y (mgr.34107) 364 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:02.520 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:02 smithi089 bash[84777]: cluster 2024-04-13T07:47:01.186661+0000 mgr.y (mgr.34107) 364 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:02.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:02 smithi203 bash[68060]: cluster 2024-04-13T07:47:01.186661+0000 mgr.y (mgr.34107) 364 : cluster [DBG] pgmap v300: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:02.767 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:47:02 smithi203 bash[60252]: ts=2024-04-13T07:47:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:47:04.430 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:47:04.430 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:47:04.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:04 smithi089 bash[82331]: cluster 2024-04-13T07:47:03.187690+0000 mgr.y (mgr.34107) 365 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:04.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:04 smithi089 bash[84777]: cluster 2024-04-13T07:47:03.187690+0000 mgr.y (mgr.34107) 365 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:04.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:04 smithi203 bash[68060]: cluster 2024-04-13T07:47:03.187690+0000 mgr.y (mgr.34107) 365 : cluster [DBG] pgmap v301: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:05.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:05 smithi089 bash[82331]: audit 2024-04-13T07:47:04.435437+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/2412290076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:05.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:05 smithi089 bash[84777]: audit 2024-04-13T07:47:04.435437+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/2412290076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:05.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:05 smithi203 bash[68060]: audit 2024-04-13T07:47:04.435437+0000 mon.c (mon.2) 31 : audit [DBG] from='client.? 172.21.15.89:0/2412290076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:06.284 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:47:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:47:05] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:47:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:06 smithi089 bash[82331]: cluster 2024-04-13T07:47:05.188419+0000 mgr.y (mgr.34107) 366 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:06.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:06 smithi089 bash[84777]: cluster 2024-04-13T07:47:05.188419+0000 mgr.y (mgr.34107) 366 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:06.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:06 smithi203 bash[68060]: cluster 2024-04-13T07:47:05.188419+0000 mgr.y (mgr.34107) 366 : cluster [DBG] pgmap v302: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:08.070 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:47:08.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:08 smithi089 bash[84777]: cluster 2024-04-13T07:47:07.189320+0000 mgr.y (mgr.34107) 367 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:08.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:08 smithi089 bash[82331]: cluster 2024-04-13T07:47:07.189320+0000 mgr.y (mgr.34107) 367 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:08.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:08 smithi203 bash[68060]: cluster 2024-04-13T07:47:07.189320+0000 mgr.y (mgr.34107) 367 : cluster [DBG] pgmap v303: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:10.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:10 smithi089 bash[84777]: cluster 2024-04-13T07:47:09.190287+0000 mgr.y (mgr.34107) 368 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:10.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:10 smithi089 bash[82331]: cluster 2024-04-13T07:47:09.190287+0000 mgr.y (mgr.34107) 368 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:10.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:10 smithi203 bash[68060]: cluster 2024-04-13T07:47:09.190287+0000 mgr.y (mgr.34107) 368 : cluster [DBG] pgmap v304: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:12.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:12 smithi089 bash[84777]: cluster 2024-04-13T07:47:11.191154+0000 mgr.y (mgr.34107) 369 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:12.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:12 smithi089 bash[84777]: audit 2024-04-13T07:47:11.358731+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:47:12.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:12 smithi089 bash[84777]: audit 2024-04-13T07:47:12.078438+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:12.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:12 smithi089 bash[82331]: cluster 2024-04-13T07:47:11.191154+0000 mgr.y (mgr.34107) 369 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:12.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:12 smithi089 bash[82331]: audit 2024-04-13T07:47:11.358731+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:47:12.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:12 smithi089 bash[82331]: audit 2024-04-13T07:47:12.078438+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:12.767 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:47:12 smithi203 bash[60252]: ts=2024-04-13T07:47:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:47:12.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:12 smithi203 bash[68060]: cluster 2024-04-13T07:47:11.191154+0000 mgr.y (mgr.34107) 369 : cluster [DBG] pgmap v305: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:12.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:12 smithi203 bash[68060]: audit 2024-04-13T07:47:11.358731+0000 mon.a (mon.0) 620 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:47:12.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:12 smithi203 bash[68060]: audit 2024-04-13T07:47:12.078438+0000 mon.a (mon.0) 621 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:12.806 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:47:14.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:14 smithi203 bash[68060]: cluster 2024-04-13T07:47:13.192109+0000 mgr.y (mgr.34107) 370 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:14.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:14 smithi089 bash[82331]: cluster 2024-04-13T07:47:13.192109+0000 mgr.y (mgr.34107) 370 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:14.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:14 smithi089 bash[84777]: cluster 2024-04-13T07:47:13.192109+0000 mgr.y (mgr.34107) 370 : cluster [DBG] pgmap v306: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:15.630 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:47:15.630 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:47:16.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:47:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:47:15] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.43.0" 2024-04-13T07:47:16.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:16 smithi203 bash[68060]: cluster 2024-04-13T07:47:15.192754+0000 mgr.y (mgr.34107) 371 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:16.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:16 smithi203 bash[68060]: audit 2024-04-13T07:47:15.635444+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/717826478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:16 smithi089 bash[82331]: cluster 2024-04-13T07:47:15.192754+0000 mgr.y (mgr.34107) 371 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:16.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:16 smithi089 bash[82331]: audit 2024-04-13T07:47:15.635444+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/717826478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:16 smithi089 bash[84777]: cluster 2024-04-13T07:47:15.192754+0000 mgr.y (mgr.34107) 371 : cluster [DBG] pgmap v307: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:16 smithi089 bash[84777]: audit 2024-04-13T07:47:15.635444+0000 mon.c (mon.2) 32 : audit [DBG] from='client.? 172.21.15.89:0/717826478' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:18.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:18 smithi203 bash[68060]: cluster 2024-04-13T07:47:17.193275+0000 mgr.y (mgr.34107) 372 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:18.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:18 smithi203 bash[68060]: audit 2024-04-13T07:47:17.398715+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:18 smithi203 bash[68060]: audit 2024-04-13T07:47:17.404033+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:18 smithi203 bash[68060]: audit 2024-04-13T07:47:17.503371+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:18 smithi203 bash[68060]: audit 2024-04-13T07:47:17.510279+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:18 smithi203 bash[68060]: audit 2024-04-13T07:47:18.042715+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:47:18.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:18 smithi203 bash[68060]: audit 2024-04-13T07:47:18.043933+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:47:18.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:18 smithi203 bash[68060]: audit 2024-04-13T07:47:18.050832+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[82331]: cluster 2024-04-13T07:47:17.193275+0000 mgr.y (mgr.34107) 372 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:18.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[82331]: audit 2024-04-13T07:47:17.398715+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[82331]: audit 2024-04-13T07:47:17.404033+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[82331]: audit 2024-04-13T07:47:17.503371+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[82331]: audit 2024-04-13T07:47:17.510279+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[82331]: audit 2024-04-13T07:47:18.042715+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[82331]: audit 2024-04-13T07:47:18.043933+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[82331]: audit 2024-04-13T07:47:18.050832+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[84777]: cluster 2024-04-13T07:47:17.193275+0000 mgr.y (mgr.34107) 372 : cluster [DBG] pgmap v308: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[84777]: audit 2024-04-13T07:47:17.398715+0000 mon.a (mon.0) 622 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[84777]: audit 2024-04-13T07:47:17.404033+0000 mon.a (mon.0) 623 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[84777]: audit 2024-04-13T07:47:17.503371+0000 mon.a (mon.0) 624 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[84777]: audit 2024-04-13T07:47:17.510279+0000 mon.a (mon.0) 625 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[84777]: audit 2024-04-13T07:47:18.042715+0000 mon.a (mon.0) 626 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:47:18.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[84777]: audit 2024-04-13T07:47:18.043933+0000 mon.a (mon.0) 627 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:47:18.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:18 smithi089 bash[84777]: audit 2024-04-13T07:47:18.050832+0000 mon.a (mon.0) 628 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:47:19.348 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:47:20.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:20 smithi203 bash[68060]: cluster 2024-04-13T07:47:19.194149+0000 mgr.y (mgr.34107) 373 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:20.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:20 smithi089 bash[84777]: cluster 2024-04-13T07:47:19.194149+0000 mgr.y (mgr.34107) 373 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:20.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:20 smithi089 bash[82331]: cluster 2024-04-13T07:47:19.194149+0000 mgr.y (mgr.34107) 373 : cluster [DBG] pgmap v309: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:22.682 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:47:22 smithi203 bash[60252]: ts=2024-04-13T07:47:22.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:47:23.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:22 smithi203 bash[68060]: cluster 2024-04-13T07:47:21.194953+0000 mgr.y (mgr.34107) 374 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:23.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:22 smithi089 bash[84777]: cluster 2024-04-13T07:47:21.194953+0000 mgr.y (mgr.34107) 374 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:23.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:22 smithi089 bash[82331]: cluster 2024-04-13T07:47:21.194953+0000 mgr.y (mgr.34107) 374 : cluster [DBG] pgmap v310: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:24.090 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:47:24.953 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:24 smithi089 bash[84777]: cluster 2024-04-13T07:47:23.195983+0000 mgr.y (mgr.34107) 375 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:24.954 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:24 smithi089 bash[82331]: cluster 2024-04-13T07:47:23.195983+0000 mgr.y (mgr.34107) 375 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:25.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:24 smithi203 bash[68060]: cluster 2024-04-13T07:47:23.195983+0000 mgr.y (mgr.34107) 375 : cluster [DBG] pgmap v311: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:26.049 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:47:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:47:25] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.43.0" 2024-04-13T07:47:26.845 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:47:26.845 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:47:27.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:26 smithi203 bash[68060]: cluster 2024-04-13T07:47:25.196495+0000 mgr.y (mgr.34107) 376 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:27.024 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:26 smithi089 bash[82331]: cluster 2024-04-13T07:47:25.196495+0000 mgr.y (mgr.34107) 376 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:27.024 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:26 smithi089 bash[84777]: cluster 2024-04-13T07:47:25.196495+0000 mgr.y (mgr.34107) 376 : cluster [DBG] pgmap v312: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:27 smithi089 bash[82331]: audit 2024-04-13T07:47:26.850387+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.89:0/3259663660' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:27 smithi089 bash[82331]: audit 2024-04-13T07:47:27.078656+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:27.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:27 smithi089 bash[84777]: audit 2024-04-13T07:47:26.850387+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.89:0/3259663660' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:27.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:27 smithi089 bash[84777]: audit 2024-04-13T07:47:27.078656+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:28.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:27 smithi203 bash[68060]: audit 2024-04-13T07:47:26.850387+0000 mon.a (mon.0) 629 : audit [DBG] from='client.? 172.21.15.89:0/3259663660' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:28.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:27 smithi203 bash[68060]: audit 2024-04-13T07:47:27.078656+0000 mon.a (mon.0) 630 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:29.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:28 smithi203 bash[68060]: cluster 2024-04-13T07:47:27.197270+0000 mgr.y (mgr.34107) 377 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:29.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:28 smithi089 bash[84777]: cluster 2024-04-13T07:47:27.197270+0000 mgr.y (mgr.34107) 377 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:29.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:28 smithi089 bash[82331]: cluster 2024-04-13T07:47:27.197270+0000 mgr.y (mgr.34107) 377 : cluster [DBG] pgmap v313: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:30.483 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:47:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:30 smithi089 bash[82331]: cluster 2024-04-13T07:47:29.198113+0000 mgr.y (mgr.34107) 378 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:30 smithi089 bash[84777]: cluster 2024-04-13T07:47:29.198113+0000 mgr.y (mgr.34107) 378 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:31.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:30 smithi203 bash[68060]: cluster 2024-04-13T07:47:29.198113+0000 mgr.y (mgr.34107) 378 : cluster [DBG] pgmap v314: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:32.712 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:47:32 smithi203 bash[60252]: ts=2024-04-13T07:47:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:47:33.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:32 smithi203 bash[68060]: cluster 2024-04-13T07:47:31.198948+0000 mgr.y (mgr.34107) 379 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:33.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:32 smithi089 bash[82331]: cluster 2024-04-13T07:47:31.198948+0000 mgr.y (mgr.34107) 379 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:33.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:32 smithi089 bash[84777]: cluster 2024-04-13T07:47:31.198948+0000 mgr.y (mgr.34107) 379 : cluster [DBG] pgmap v315: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:35.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:34 smithi203 bash[68060]: cluster 2024-04-13T07:47:33.199599+0000 mgr.y (mgr.34107) 380 : cluster [DBG] pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:35.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:34 smithi089 bash[82331]: cluster 2024-04-13T07:47:33.199599+0000 mgr.y (mgr.34107) 380 : cluster [DBG] pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:35.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:34 smithi089 bash[84777]: cluster 2024-04-13T07:47:33.199599+0000 mgr.y (mgr.34107) 380 : cluster [DBG] pgmap v316: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:35.224 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:47:36.126 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:47:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:47:35] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.43.0" 2024-04-13T07:47:37.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:36 smithi203 bash[68060]: cluster 2024-04-13T07:47:35.200401+0000 mgr.y (mgr.34107) 381 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:37.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:36 smithi089 bash[82331]: cluster 2024-04-13T07:47:35.200401+0000 mgr.y (mgr.34107) 381 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:37.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:36 smithi089 bash[84777]: cluster 2024-04-13T07:47:35.200401+0000 mgr.y (mgr.34107) 381 : cluster [DBG] pgmap v317: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:38.086 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:47:38.086 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:47:39.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:38 smithi203 bash[68060]: cluster 2024-04-13T07:47:37.201351+0000 mgr.y (mgr.34107) 382 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:39.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:38 smithi203 bash[68060]: audit 2024-04-13T07:47:38.092035+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/3515567365' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:39.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:38 smithi089 bash[84777]: cluster 2024-04-13T07:47:37.201351+0000 mgr.y (mgr.34107) 382 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:39.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:38 smithi089 bash[84777]: audit 2024-04-13T07:47:38.092035+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/3515567365' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:39.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:38 smithi089 bash[82331]: cluster 2024-04-13T07:47:37.201351+0000 mgr.y (mgr.34107) 382 : cluster [DBG] pgmap v318: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:39.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:38 smithi089 bash[82331]: audit 2024-04-13T07:47:38.092035+0000 mon.c (mon.2) 33 : audit [DBG] from='client.? 172.21.15.89:0/3515567365' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:41.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:40 smithi203 bash[68060]: cluster 2024-04-13T07:47:39.202289+0000 mgr.y (mgr.34107) 383 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:41.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:40 smithi089 bash[82331]: cluster 2024-04-13T07:47:39.202289+0000 mgr.y (mgr.34107) 383 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:41.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:40 smithi089 bash[84777]: cluster 2024-04-13T07:47:39.202289+0000 mgr.y (mgr.34107) 383 : cluster [DBG] pgmap v319: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:41.704 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:47:42.732 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:47:42 smithi203 bash[60252]: ts=2024-04-13T07:47:42.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:47:43.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:42 smithi203 bash[68060]: cluster 2024-04-13T07:47:41.203133+0000 mgr.y (mgr.34107) 384 : cluster [DBG] pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:43.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:42 smithi203 bash[68060]: audit 2024-04-13T07:47:42.079319+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:43.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:42 smithi089 bash[84777]: cluster 2024-04-13T07:47:41.203133+0000 mgr.y (mgr.34107) 384 : cluster [DBG] pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:43.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:42 smithi089 bash[84777]: audit 2024-04-13T07:47:42.079319+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:43.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:42 smithi089 bash[82331]: cluster 2024-04-13T07:47:41.203133+0000 mgr.y (mgr.34107) 384 : cluster [DBG] pgmap v320: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:43.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:42 smithi089 bash[82331]: audit 2024-04-13T07:47:42.079319+0000 mon.a (mon.0) 631 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:45.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:44 smithi203 bash[68060]: cluster 2024-04-13T07:47:43.204061+0000 mgr.y (mgr.34107) 385 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:45.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:44 smithi089 bash[84777]: cluster 2024-04-13T07:47:43.204061+0000 mgr.y (mgr.34107) 385 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:45.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:44 smithi089 bash[82331]: cluster 2024-04-13T07:47:43.204061+0000 mgr.y (mgr.34107) 385 : cluster [DBG] pgmap v321: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:46.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:47:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:47:45] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:47:46.437 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:47:47.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:46 smithi203 bash[68060]: cluster 2024-04-13T07:47:45.204833+0000 mgr.y (mgr.34107) 386 : cluster [DBG] pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:47.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:46 smithi089 bash[84777]: cluster 2024-04-13T07:47:45.204833+0000 mgr.y (mgr.34107) 386 : cluster [DBG] pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:47.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:46 smithi089 bash[82331]: cluster 2024-04-13T07:47:45.204833+0000 mgr.y (mgr.34107) 386 : cluster [DBG] pgmap v322: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:49.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:48 smithi203 bash[68060]: cluster 2024-04-13T07:47:47.205638+0000 mgr.y (mgr.34107) 387 : cluster [DBG] pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:49.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:48 smithi089 bash[82331]: cluster 2024-04-13T07:47:47.205638+0000 mgr.y (mgr.34107) 387 : cluster [DBG] pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:49.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:48 smithi089 bash[84777]: cluster 2024-04-13T07:47:47.205638+0000 mgr.y (mgr.34107) 387 : cluster [DBG] pgmap v323: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:49.326 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:47:49.326 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:47:50.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:49 smithi203 bash[68060]: audit 2024-04-13T07:47:49.331596+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.89:0/1316805994' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:50.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:49 smithi089 bash[82331]: audit 2024-04-13T07:47:49.331596+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.89:0/1316805994' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:50.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:49 smithi089 bash[84777]: audit 2024-04-13T07:47:49.331596+0000 mon.a (mon.0) 632 : audit [DBG] from='client.? 172.21.15.89:0/1316805994' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:47:51.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:50 smithi203 bash[68060]: cluster 2024-04-13T07:47:49.206489+0000 mgr.y (mgr.34107) 388 : cluster [DBG] pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:51.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:50 smithi089 bash[82331]: cluster 2024-04-13T07:47:49.206489+0000 mgr.y (mgr.34107) 388 : cluster [DBG] pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:51.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:50 smithi089 bash[84777]: cluster 2024-04-13T07:47:49.206489+0000 mgr.y (mgr.34107) 388 : cluster [DBG] pgmap v324: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:52.761 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:47:52 smithi203 bash[60252]: ts=2024-04-13T07:47:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:47:53.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:52 smithi203 bash[68060]: cluster 2024-04-13T07:47:51.207320+0000 mgr.y (mgr.34107) 389 : cluster [DBG] pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:53.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:52 smithi089 bash[84777]: cluster 2024-04-13T07:47:51.207320+0000 mgr.y (mgr.34107) 389 : cluster [DBG] pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:53.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:52 smithi089 bash[82331]: cluster 2024-04-13T07:47:51.207320+0000 mgr.y (mgr.34107) 389 : cluster [DBG] pgmap v325: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:53.059 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:47:55.017 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:54 smithi203 bash[68060]: cluster 2024-04-13T07:47:53.207972+0000 mgr.y (mgr.34107) 390 : cluster [DBG] pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:55.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:54 smithi089 bash[82331]: cluster 2024-04-13T07:47:53.207972+0000 mgr.y (mgr.34107) 390 : cluster [DBG] pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:55.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:54 smithi089 bash[84777]: cluster 2024-04-13T07:47:53.207972+0000 mgr.y (mgr.34107) 390 : cluster [DBG] pgmap v326: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:56.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:47:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:47:55] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:47:57.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:56 smithi089 bash[82331]: cluster 2024-04-13T07:47:55.208665+0000 mgr.y (mgr.34107) 391 : cluster [DBG] pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:57.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:56 smithi089 bash[84777]: cluster 2024-04-13T07:47:55.208665+0000 mgr.y (mgr.34107) 391 : cluster [DBG] pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:57.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:56 smithi203 bash[68060]: cluster 2024-04-13T07:47:55.208665+0000 mgr.y (mgr.34107) 391 : cluster [DBG] pgmap v327: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:57.799 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:47:58.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:57 smithi089 bash[82331]: audit 2024-04-13T07:47:57.079705+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:58.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:57 smithi089 bash[84777]: audit 2024-04-13T07:47:57.079705+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:58.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:57 smithi203 bash[68060]: audit 2024-04-13T07:47:57.079705+0000 mon.a (mon.0) 633 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:47:58.977 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:47:58 smithi089 bash[84777]: cluster 2024-04-13T07:47:57.209444+0000 mgr.y (mgr.34107) 392 : cluster [DBG] pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:58.978 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:47:58 smithi089 bash[82331]: cluster 2024-04-13T07:47:57.209444+0000 mgr.y (mgr.34107) 392 : cluster [DBG] pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:47:59.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:47:58 smithi203 bash[68060]: cluster 2024-04-13T07:47:57.209444+0000 mgr.y (mgr.34107) 392 : cluster [DBG] pgmap v328: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:00.546 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:48:00.547 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:48:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:00 smithi089 bash[82331]: cluster 2024-04-13T07:47:59.210293+0000 mgr.y (mgr.34107) 393 : cluster [DBG] pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:01.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:00 smithi089 bash[82331]: audit 2024-04-13T07:48:00.552129+0000 mon.a (mon.0) 634 : audit [DBG] from='client.? 172.21.15.89:0/2891083843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:00 smithi089 bash[84777]: cluster 2024-04-13T07:47:59.210293+0000 mgr.y (mgr.34107) 393 : cluster [DBG] pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:01.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:00 smithi089 bash[84777]: audit 2024-04-13T07:48:00.552129+0000 mon.a (mon.0) 634 : audit [DBG] from='client.? 172.21.15.89:0/2891083843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:01.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:00 smithi203 bash[68060]: cluster 2024-04-13T07:47:59.210293+0000 mgr.y (mgr.34107) 393 : cluster [DBG] pgmap v329: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:01.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:00 smithi203 bash[68060]: audit 2024-04-13T07:48:00.552129+0000 mon.a (mon.0) 634 : audit [DBG] from='client.? 172.21.15.89:0/2891083843' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:02.767 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:48:02 smithi203 bash[60252]: ts=2024-04-13T07:48:02.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:48:03.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:02 smithi089 bash[82331]: cluster 2024-04-13T07:48:01.211175+0000 mgr.y (mgr.34107) 394 : cluster [DBG] pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:03.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:02 smithi089 bash[84777]: cluster 2024-04-13T07:48:01.211175+0000 mgr.y (mgr.34107) 394 : cluster [DBG] pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:03.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:02 smithi203 bash[68060]: cluster 2024-04-13T07:48:01.211175+0000 mgr.y (mgr.34107) 394 : cluster [DBG] pgmap v330: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:04.185 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:48:05.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:04 smithi089 bash[84777]: cluster 2024-04-13T07:48:03.212093+0000 mgr.y (mgr.34107) 395 : cluster [DBG] pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:05.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:04 smithi089 bash[82331]: cluster 2024-04-13T07:48:03.212093+0000 mgr.y (mgr.34107) 395 : cluster [DBG] pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:05.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:04 smithi203 bash[68060]: cluster 2024-04-13T07:48:03.212093+0000 mgr.y (mgr.34107) 395 : cluster [DBG] pgmap v331: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:06.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:48:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:48:05] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.43.0" 2024-04-13T07:48:07.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:06 smithi089 bash[82331]: cluster 2024-04-13T07:48:05.212887+0000 mgr.y (mgr.34107) 396 : cluster [DBG] pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:07.050 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:06 smithi089 bash[84777]: cluster 2024-04-13T07:48:05.212887+0000 mgr.y (mgr.34107) 396 : cluster [DBG] pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:07.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:06 smithi203 bash[68060]: cluster 2024-04-13T07:48:05.212887+0000 mgr.y (mgr.34107) 396 : cluster [DBG] pgmap v332: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:08.927 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:48:09.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:08 smithi203 bash[68060]: cluster 2024-04-13T07:48:07.213797+0000 mgr.y (mgr.34107) 397 : cluster [DBG] pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:09.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:08 smithi089 bash[82331]: cluster 2024-04-13T07:48:07.213797+0000 mgr.y (mgr.34107) 397 : cluster [DBG] pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:09.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:08 smithi089 bash[84777]: cluster 2024-04-13T07:48:07.213797+0000 mgr.y (mgr.34107) 397 : cluster [DBG] pgmap v333: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:11.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:10 smithi089 bash[84777]: cluster 2024-04-13T07:48:09.214739+0000 mgr.y (mgr.34107) 398 : cluster [DBG] pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:11.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:10 smithi089 bash[82331]: cluster 2024-04-13T07:48:09.214739+0000 mgr.y (mgr.34107) 398 : cluster [DBG] pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:11.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:10 smithi203 bash[68060]: cluster 2024-04-13T07:48:09.214739+0000 mgr.y (mgr.34107) 398 : cluster [DBG] pgmap v334: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:11.721 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:48:11.721 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:48:12.049 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:11 smithi089 bash[82331]: audit 2024-04-13T07:48:11.726765+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.89:0/3735796157' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:12.049 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:11 smithi089 bash[84777]: audit 2024-04-13T07:48:11.726765+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.89:0/3735796157' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:12.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:11 smithi203 bash[68060]: audit 2024-04-13T07:48:11.726765+0000 mon.a (mon.0) 635 : audit [DBG] from='client.? 172.21.15.89:0/3735796157' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:12.767 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:48:12 smithi203 bash[60252]: ts=2024-04-13T07:48:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:48:13.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:12 smithi203 bash[68060]: cluster 2024-04-13T07:48:11.215600+0000 mgr.y (mgr.34107) 399 : cluster [DBG] pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:13.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:12 smithi203 bash[68060]: audit 2024-04-13T07:48:12.080198+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:13.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:12 smithi089 bash[82331]: cluster 2024-04-13T07:48:11.215600+0000 mgr.y (mgr.34107) 399 : cluster [DBG] pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:13.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:12 smithi089 bash[82331]: audit 2024-04-13T07:48:12.080198+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:13.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:12 smithi089 bash[84777]: cluster 2024-04-13T07:48:11.215600+0000 mgr.y (mgr.34107) 399 : cluster [DBG] pgmap v335: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:13.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:12 smithi089 bash[84777]: audit 2024-04-13T07:48:12.080198+0000 mon.a (mon.0) 636 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:14.267 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:13 smithi203 bash[68060]: cluster 2024-04-13T07:48:13.216561+0000 mgr.y (mgr.34107) 400 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:14.299 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:13 smithi089 bash[82331]: cluster 2024-04-13T07:48:13.216561+0000 mgr.y (mgr.34107) 400 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:14.299 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:13 smithi089 bash[84777]: cluster 2024-04-13T07:48:13.216561+0000 mgr.y (mgr.34107) 400 : cluster [DBG] pgmap v336: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:15.385 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:48:16.268 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:48:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:48:15] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:48:16.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:16 smithi089 bash[84777]: cluster 2024-04-13T07:48:15.217331+0000 mgr.y (mgr.34107) 401 : cluster [DBG] pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:16.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:16 smithi089 bash[82331]: cluster 2024-04-13T07:48:15.217331+0000 mgr.y (mgr.34107) 401 : cluster [DBG] pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:16.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:16 smithi203 bash[68060]: cluster 2024-04-13T07:48:15.217331+0000 mgr.y (mgr.34107) 401 : cluster [DBG] pgmap v337: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:18.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:18 smithi089 bash[84777]: cluster 2024-04-13T07:48:17.218162+0000 mgr.y (mgr.34107) 402 : cluster [DBG] pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:18.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:18 smithi089 bash[84777]: audit 2024-04-13T07:48:18.126872+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:48:18.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:18 smithi089 bash[82331]: cluster 2024-04-13T07:48:17.218162+0000 mgr.y (mgr.34107) 402 : cluster [DBG] pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:18.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:18 smithi089 bash[82331]: audit 2024-04-13T07:48:18.126872+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:48:18.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:18 smithi203 bash[68060]: cluster 2024-04-13T07:48:17.218162+0000 mgr.y (mgr.34107) 402 : cluster [DBG] pgmap v338: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:18.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:18 smithi203 bash[68060]: audit 2024-04-13T07:48:18.126872+0000 mon.a (mon.0) 637 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:48:20.111 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:48:20.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:20 smithi089 bash[82331]: cluster 2024-04-13T07:48:19.219108+0000 mgr.y (mgr.34107) 403 : cluster [DBG] pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:20.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:20 smithi089 bash[84777]: cluster 2024-04-13T07:48:19.219108+0000 mgr.y (mgr.34107) 403 : cluster [DBG] pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:20.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:20 smithi203 bash[68060]: cluster 2024-04-13T07:48:19.219108+0000 mgr.y (mgr.34107) 403 : cluster [DBG] pgmap v339: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:22.279 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:22 smithi089 bash[82331]: cluster 2024-04-13T07:48:21.219731+0000 mgr.y (mgr.34107) 404 : cluster [DBG] pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:22.530 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:22 smithi089 bash[84777]: cluster 2024-04-13T07:48:21.219731+0000 mgr.y (mgr.34107) 404 : cluster [DBG] pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:22.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:22 smithi203 bash[68060]: cluster 2024-04-13T07:48:21.219731+0000 mgr.y (mgr.34107) 404 : cluster [DBG] pgmap v340: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:22.768 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:48:22 smithi203 bash[60252]: ts=2024-04-13T07:48:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:48:22.964 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:48:22.964 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:48:23.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:23 smithi089 bash[84777]: audit 2024-04-13T07:48:22.969604+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/3470418437' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:23.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:23 smithi089 bash[84777]: audit 2024-04-13T07:48:23.009221+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:23.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:23 smithi089 bash[84777]: audit 2024-04-13T07:48:23.016478+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:23.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:23 smithi089 bash[82331]: audit 2024-04-13T07:48:22.969604+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/3470418437' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:23.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:23 smithi089 bash[82331]: audit 2024-04-13T07:48:23.009221+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:23.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:23 smithi089 bash[82331]: audit 2024-04-13T07:48:23.016478+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:23.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:23 smithi203 bash[68060]: audit 2024-04-13T07:48:22.969604+0000 mon.c (mon.2) 34 : audit [DBG] from='client.? 172.21.15.89:0/3470418437' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:23.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:23 smithi203 bash[68060]: audit 2024-04-13T07:48:23.009221+0000 mon.a (mon.0) 638 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:23.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:23 smithi203 bash[68060]: audit 2024-04-13T07:48:23.016478+0000 mon.a (mon.0) 639 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:24.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:24 smithi089 bash[82331]: cluster 2024-04-13T07:48:23.220621+0000 mgr.y (mgr.34107) 405 : cluster [DBG] pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:24.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:24 smithi089 bash[82331]: audit 2024-04-13T07:48:24.124451+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:24.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:24 smithi089 bash[82331]: audit 2024-04-13T07:48:24.138729+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:24.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:24 smithi089 bash[84777]: cluster 2024-04-13T07:48:23.220621+0000 mgr.y (mgr.34107) 405 : cluster [DBG] pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:24.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:24 smithi089 bash[84777]: audit 2024-04-13T07:48:24.124451+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:24.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:24 smithi089 bash[84777]: audit 2024-04-13T07:48:24.138729+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:24.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:24 smithi203 bash[68060]: cluster 2024-04-13T07:48:23.220621+0000 mgr.y (mgr.34107) 405 : cluster [DBG] pgmap v341: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:24.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:24 smithi203 bash[68060]: audit 2024-04-13T07:48:24.124451+0000 mon.a (mon.0) 640 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:24.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:24 smithi203 bash[68060]: audit 2024-04-13T07:48:24.138729+0000 mon.a (mon.0) 641 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:25.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:25 smithi089 bash[82331]: audit 2024-04-13T07:48:24.820447+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:48:25.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:25 smithi089 bash[82331]: audit 2024-04-13T07:48:24.822304+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:48:25.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:25 smithi089 bash[82331]: audit 2024-04-13T07:48:24.831691+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:25.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:25 smithi089 bash[84777]: audit 2024-04-13T07:48:24.820447+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:48:25.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:25 smithi089 bash[84777]: audit 2024-04-13T07:48:24.822304+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:48:25.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:25 smithi089 bash[84777]: audit 2024-04-13T07:48:24.831691+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:25.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:25 smithi203 bash[68060]: audit 2024-04-13T07:48:24.820447+0000 mon.a (mon.0) 642 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:48:25.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:25 smithi203 bash[68060]: audit 2024-04-13T07:48:24.822304+0000 mon.a (mon.0) 643 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:48:25.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:25 smithi203 bash[68060]: audit 2024-04-13T07:48:24.831691+0000 mon.a (mon.0) 644 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:48:26.294 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:48:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:48:25] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:48:26.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:26 smithi089 bash[82331]: cluster 2024-04-13T07:48:25.221370+0000 mgr.y (mgr.34107) 406 : cluster [DBG] pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:26.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:26 smithi089 bash[84777]: cluster 2024-04-13T07:48:25.221370+0000 mgr.y (mgr.34107) 406 : cluster [DBG] pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:26.572 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:48:26.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:26 smithi203 bash[68060]: cluster 2024-04-13T07:48:25.221370+0000 mgr.y (mgr.34107) 406 : cluster [DBG] pgmap v342: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:27.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:27 smithi203 bash[68060]: audit 2024-04-13T07:48:27.080886+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:27.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:27 smithi089 bash[84777]: audit 2024-04-13T07:48:27.080886+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:27.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:27 smithi089 bash[82331]: audit 2024-04-13T07:48:27.080886+0000 mon.a (mon.0) 645 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:28.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:28 smithi203 bash[68060]: cluster 2024-04-13T07:48:27.222106+0000 mgr.y (mgr.34107) 407 : cluster [DBG] pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:28.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:28 smithi089 bash[84777]: cluster 2024-04-13T07:48:27.222106+0000 mgr.y (mgr.34107) 407 : cluster [DBG] pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:28.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:28 smithi089 bash[82331]: cluster 2024-04-13T07:48:27.222106+0000 mgr.y (mgr.34107) 407 : cluster [DBG] pgmap v343: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:30.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:30 smithi203 bash[68060]: cluster 2024-04-13T07:48:29.223074+0000 mgr.y (mgr.34107) 408 : cluster [DBG] pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:30.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:30 smithi089 bash[82331]: cluster 2024-04-13T07:48:29.223074+0000 mgr.y (mgr.34107) 408 : cluster [DBG] pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:30.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:30 smithi089 bash[84777]: cluster 2024-04-13T07:48:29.223074+0000 mgr.y (mgr.34107) 408 : cluster [DBG] pgmap v344: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:31.317 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:48:32.547 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:32 smithi089 bash[84777]: cluster 2024-04-13T07:48:31.223863+0000 mgr.y (mgr.34107) 409 : cluster [DBG] pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:32.547 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:32 smithi089 bash[82331]: cluster 2024-04-13T07:48:31.223863+0000 mgr.y (mgr.34107) 409 : cluster [DBG] pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:32.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:32 smithi203 bash[68060]: cluster 2024-04-13T07:48:31.223863+0000 mgr.y (mgr.34107) 409 : cluster [DBG] pgmap v345: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:32.768 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:48:32 smithi203 bash[60252]: ts=2024-04-13T07:48:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:48:34.224 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:48:34.224 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:48:34.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:34 smithi089 bash[82331]: cluster 2024-04-13T07:48:33.224843+0000 mgr.y (mgr.34107) 410 : cluster [DBG] pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:34.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:34 smithi089 bash[82331]: audit 2024-04-13T07:48:34.229676+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/2962060481' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:34.549 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:34 smithi089 bash[84777]: cluster 2024-04-13T07:48:33.224843+0000 mgr.y (mgr.34107) 410 : cluster [DBG] pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:34.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:34 smithi089 bash[84777]: audit 2024-04-13T07:48:34.229676+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/2962060481' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:34.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:34 smithi203 bash[68060]: cluster 2024-04-13T07:48:33.224843+0000 mgr.y (mgr.34107) 410 : cluster [DBG] pgmap v346: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:34.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:34 smithi203 bash[68060]: audit 2024-04-13T07:48:34.229676+0000 mon.c (mon.2) 35 : audit [DBG] from='client.? 172.21.15.89:0/2962060481' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:36.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:48:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:48:35] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.43.0" 2024-04-13T07:48:36.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:36 smithi203 bash[68060]: cluster 2024-04-13T07:48:35.225586+0000 mgr.y (mgr.34107) 411 : cluster [DBG] pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:36.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:36 smithi089 bash[82331]: cluster 2024-04-13T07:48:35.225586+0000 mgr.y (mgr.34107) 411 : cluster [DBG] pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:36.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:36 smithi089 bash[84777]: cluster 2024-04-13T07:48:35.225586+0000 mgr.y (mgr.34107) 411 : cluster [DBG] pgmap v347: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:37.886 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:48:38.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:38 smithi203 bash[68060]: cluster 2024-04-13T07:48:37.226412+0000 mgr.y (mgr.34107) 412 : cluster [DBG] pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:38.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:38 smithi089 bash[82331]: cluster 2024-04-13T07:48:37.226412+0000 mgr.y (mgr.34107) 412 : cluster [DBG] pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:38.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:38 smithi089 bash[84777]: cluster 2024-04-13T07:48:37.226412+0000 mgr.y (mgr.34107) 412 : cluster [DBG] pgmap v348: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:40.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:40 smithi203 bash[68060]: cluster 2024-04-13T07:48:39.227287+0000 mgr.y (mgr.34107) 413 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:40.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:40 smithi089 bash[82331]: cluster 2024-04-13T07:48:39.227287+0000 mgr.y (mgr.34107) 413 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:40.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:40 smithi089 bash[84777]: cluster 2024-04-13T07:48:39.227287+0000 mgr.y (mgr.34107) 413 : cluster [DBG] pgmap v349: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:42.625 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:48:42.648 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:42 smithi089 bash[84777]: cluster 2024-04-13T07:48:41.228065+0000 mgr.y (mgr.34107) 414 : cluster [DBG] pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:42.649 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:42 smithi089 bash[84777]: audit 2024-04-13T07:48:42.081164+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:42.649 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:42 smithi089 bash[82331]: cluster 2024-04-13T07:48:41.228065+0000 mgr.y (mgr.34107) 414 : cluster [DBG] pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:42.649 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:42 smithi089 bash[82331]: audit 2024-04-13T07:48:42.081164+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:42.768 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:48:42 smithi203 bash[60252]: ts=2024-04-13T07:48:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:48:42.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:42 smithi203 bash[68060]: cluster 2024-04-13T07:48:41.228065+0000 mgr.y (mgr.34107) 414 : cluster [DBG] pgmap v350: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:42.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:42 smithi203 bash[68060]: audit 2024-04-13T07:48:42.081164+0000 mon.a (mon.0) 646 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:44.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:44 smithi089 bash[82331]: cluster 2024-04-13T07:48:43.228945+0000 mgr.y (mgr.34107) 415 : cluster [DBG] pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:44.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:44 smithi089 bash[84777]: cluster 2024-04-13T07:48:43.228945+0000 mgr.y (mgr.34107) 415 : cluster [DBG] pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:44.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:44 smithi203 bash[68060]: cluster 2024-04-13T07:48:43.228945+0000 mgr.y (mgr.34107) 415 : cluster [DBG] pgmap v351: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:45.455 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:48:45.455 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:48:46.110 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:48:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:48:45] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.43.0" 2024-04-13T07:48:46.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:46 smithi089 bash[82331]: cluster 2024-04-13T07:48:45.229375+0000 mgr.y (mgr.34107) 416 : cluster [DBG] pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:46.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:46 smithi089 bash[82331]: audit 2024-04-13T07:48:45.460668+0000 mon.a (mon.0) 647 : audit [DBG] from='client.? 172.21.15.89:0/2584809408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:46.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:46 smithi089 bash[84777]: cluster 2024-04-13T07:48:45.229375+0000 mgr.y (mgr.34107) 416 : cluster [DBG] pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:46.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:46 smithi089 bash[84777]: audit 2024-04-13T07:48:45.460668+0000 mon.a (mon.0) 647 : audit [DBG] from='client.? 172.21.15.89:0/2584809408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:46.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:46 smithi203 bash[68060]: cluster 2024-04-13T07:48:45.229375+0000 mgr.y (mgr.34107) 416 : cluster [DBG] pgmap v352: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:46.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:46 smithi203 bash[68060]: audit 2024-04-13T07:48:45.460668+0000 mon.a (mon.0) 647 : audit [DBG] from='client.? 172.21.15.89:0/2584809408' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:48.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:48 smithi203 bash[68060]: cluster 2024-04-13T07:48:47.230139+0000 mgr.y (mgr.34107) 417 : cluster [DBG] pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:48.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:48 smithi089 bash[84777]: cluster 2024-04-13T07:48:47.230139+0000 mgr.y (mgr.34107) 417 : cluster [DBG] pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:48.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:48 smithi089 bash[82331]: cluster 2024-04-13T07:48:47.230139+0000 mgr.y (mgr.34107) 417 : cluster [DBG] pgmap v353: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:49.112 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:48:50.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:50 smithi203 bash[68060]: cluster 2024-04-13T07:48:49.230754+0000 mgr.y (mgr.34107) 418 : cluster [DBG] pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:50.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:50 smithi089 bash[82331]: cluster 2024-04-13T07:48:49.230754+0000 mgr.y (mgr.34107) 418 : cluster [DBG] pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:50.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:50 smithi089 bash[84777]: cluster 2024-04-13T07:48:49.230754+0000 mgr.y (mgr.34107) 418 : cluster [DBG] pgmap v354: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:52.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:52 smithi203 bash[68060]: cluster 2024-04-13T07:48:51.231556+0000 mgr.y (mgr.34107) 419 : cluster [DBG] pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:52.768 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:48:52 smithi203 bash[60252]: ts=2024-04-13T07:48:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:48:52.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:52 smithi089 bash[82331]: cluster 2024-04-13T07:48:51.231556+0000 mgr.y (mgr.34107) 419 : cluster [DBG] pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:52.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:52 smithi089 bash[84777]: cluster 2024-04-13T07:48:51.231556+0000 mgr.y (mgr.34107) 419 : cluster [DBG] pgmap v355: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:53.856 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:48:54.711 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:54 smithi089 bash[82331]: cluster 2024-04-13T07:48:53.232504+0000 mgr.y (mgr.34107) 420 : cluster [DBG] pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:54.757 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:54 smithi089 bash[84777]: cluster 2024-04-13T07:48:53.232504+0000 mgr.y (mgr.34107) 420 : cluster [DBG] pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:54.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:54 smithi203 bash[68060]: cluster 2024-04-13T07:48:53.232504+0000 mgr.y (mgr.34107) 420 : cluster [DBG] pgmap v356: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:56.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:48:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:48:55] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.43.0" 2024-04-13T07:48:56.710 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:48:56.710 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:48:56.723 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:56 smithi089 bash[82331]: cluster 2024-04-13T07:48:55.233193+0000 mgr.y (mgr.34107) 421 : cluster [DBG] pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:56.723 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:56 smithi089 bash[84777]: cluster 2024-04-13T07:48:55.233193+0000 mgr.y (mgr.34107) 421 : cluster [DBG] pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:56.767 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:56 smithi203 bash[68060]: cluster 2024-04-13T07:48:55.233193+0000 mgr.y (mgr.34107) 421 : cluster [DBG] pgmap v357: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:57.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:57 smithi203 bash[68060]: audit 2024-04-13T07:48:56.715426+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/3455187377' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:57.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:57 smithi203 bash[68060]: audit 2024-04-13T07:48:57.081395+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:57.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:57 smithi089 bash[82331]: audit 2024-04-13T07:48:56.715426+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/3455187377' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:57.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:57 smithi089 bash[82331]: audit 2024-04-13T07:48:57.081395+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:57.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:57 smithi089 bash[84777]: audit 2024-04-13T07:48:56.715426+0000 mon.c (mon.2) 36 : audit [DBG] from='client.? 172.21.15.89:0/3455187377' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:48:57.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:57 smithi089 bash[84777]: audit 2024-04-13T07:48:57.081395+0000 mon.a (mon.0) 648 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:48:58.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:48:58 smithi203 bash[68060]: cluster 2024-04-13T07:48:57.234013+0000 mgr.y (mgr.34107) 422 : cluster [DBG] pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:58.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:48:58 smithi089 bash[84777]: cluster 2024-04-13T07:48:57.234013+0000 mgr.y (mgr.34107) 422 : cluster [DBG] pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:48:58.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:48:58 smithi089 bash[82331]: cluster 2024-04-13T07:48:57.234013+0000 mgr.y (mgr.34107) 422 : cluster [DBG] pgmap v358: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:00.388 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:49:00.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:00 smithi203 bash[68060]: cluster 2024-04-13T07:48:59.234882+0000 mgr.y (mgr.34107) 423 : cluster [DBG] pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:00.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:00 smithi089 bash[84777]: cluster 2024-04-13T07:48:59.234882+0000 mgr.y (mgr.34107) 423 : cluster [DBG] pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:00.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:00 smithi089 bash[82331]: cluster 2024-04-13T07:48:59.234882+0000 mgr.y (mgr.34107) 423 : cluster [DBG] pgmap v359: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:02.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:02 smithi203 bash[68060]: cluster 2024-04-13T07:49:01.235674+0000 mgr.y (mgr.34107) 424 : cluster [DBG] pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:02.768 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:49:02 smithi203 bash[60252]: ts=2024-04-13T07:49:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:49:02.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:02 smithi089 bash[82331]: cluster 2024-04-13T07:49:01.235674+0000 mgr.y (mgr.34107) 424 : cluster [DBG] pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:02.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:02 smithi089 bash[84777]: cluster 2024-04-13T07:49:01.235674+0000 mgr.y (mgr.34107) 424 : cluster [DBG] pgmap v360: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:04.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:04 smithi203 bash[68060]: cluster 2024-04-13T07:49:03.236637+0000 mgr.y (mgr.34107) 425 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:04.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:04 smithi089 bash[82331]: cluster 2024-04-13T07:49:03.236637+0000 mgr.y (mgr.34107) 425 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:04.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:04 smithi089 bash[84777]: cluster 2024-04-13T07:49:03.236637+0000 mgr.y (mgr.34107) 425 : cluster [DBG] pgmap v361: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:05.120 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:49:06.241 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:49:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:49:05] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:49:06.549 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:06 smithi089 bash[82331]: cluster 2024-04-13T07:49:05.237337+0000 mgr.y (mgr.34107) 426 : cluster [DBG] pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:06.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:06 smithi089 bash[84777]: cluster 2024-04-13T07:49:05.237337+0000 mgr.y (mgr.34107) 426 : cluster [DBG] pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:06.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:06 smithi203 bash[68060]: cluster 2024-04-13T07:49:05.237337+0000 mgr.y (mgr.34107) 426 : cluster [DBG] pgmap v362: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:07.947 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:49:07.947 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:49:08.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:08 smithi203 bash[68060]: cluster 2024-04-13T07:49:07.238074+0000 mgr.y (mgr.34107) 427 : cluster [DBG] pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:08.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:08 smithi203 bash[68060]: audit 2024-04-13T07:49:07.952552+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/4023427818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:08.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:08 smithi089 bash[82331]: cluster 2024-04-13T07:49:07.238074+0000 mgr.y (mgr.34107) 427 : cluster [DBG] pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:08.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:08 smithi089 bash[82331]: audit 2024-04-13T07:49:07.952552+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/4023427818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:08.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:08 smithi089 bash[84777]: cluster 2024-04-13T07:49:07.238074+0000 mgr.y (mgr.34107) 427 : cluster [DBG] pgmap v363: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:08.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:08 smithi089 bash[84777]: audit 2024-04-13T07:49:07.952552+0000 mon.c (mon.2) 37 : audit [DBG] from='client.? 172.21.15.89:0/4023427818' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:10.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:10 smithi203 bash[68060]: cluster 2024-04-13T07:49:09.239013+0000 mgr.y (mgr.34107) 428 : cluster [DBG] pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:10.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:10 smithi089 bash[82331]: cluster 2024-04-13T07:49:09.239013+0000 mgr.y (mgr.34107) 428 : cluster [DBG] pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:10.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:10 smithi089 bash[84777]: cluster 2024-04-13T07:49:09.239013+0000 mgr.y (mgr.34107) 428 : cluster [DBG] pgmap v364: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:11.576 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:49:12.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:12 smithi203 bash[68060]: cluster 2024-04-13T07:49:11.239728+0000 mgr.y (mgr.34107) 429 : cluster [DBG] pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:12.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:12 smithi203 bash[68060]: audit 2024-04-13T07:49:12.082007+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:12.769 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:49:12 smithi203 bash[60252]: ts=2024-04-13T07:49:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:49:12.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:12 smithi089 bash[84777]: cluster 2024-04-13T07:49:11.239728+0000 mgr.y (mgr.34107) 429 : cluster [DBG] pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:12.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:12 smithi089 bash[84777]: audit 2024-04-13T07:49:12.082007+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:12.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:12 smithi089 bash[82331]: cluster 2024-04-13T07:49:11.239728+0000 mgr.y (mgr.34107) 429 : cluster [DBG] pgmap v365: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:12.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:12 smithi089 bash[82331]: audit 2024-04-13T07:49:12.082007+0000 mon.a (mon.0) 649 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:14.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:14 smithi203 bash[68060]: cluster 2024-04-13T07:49:13.240686+0000 mgr.y (mgr.34107) 430 : cluster [DBG] pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:14.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:14 smithi089 bash[84777]: cluster 2024-04-13T07:49:13.240686+0000 mgr.y (mgr.34107) 430 : cluster [DBG] pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:14.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:14 smithi089 bash[82331]: cluster 2024-04-13T07:49:13.240686+0000 mgr.y (mgr.34107) 430 : cluster [DBG] pgmap v366: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:16.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:49:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:49:15] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:49:16.315 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:49:16.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:16 smithi203 bash[68060]: cluster 2024-04-13T07:49:15.241515+0000 mgr.y (mgr.34107) 431 : cluster [DBG] pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:16.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:16 smithi089 bash[84777]: cluster 2024-04-13T07:49:15.241515+0000 mgr.y (mgr.34107) 431 : cluster [DBG] pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:16.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:16 smithi089 bash[82331]: cluster 2024-04-13T07:49:15.241515+0000 mgr.y (mgr.34107) 431 : cluster [DBG] pgmap v367: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:18.709 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:18 smithi089 bash[84777]: cluster 2024-04-13T07:49:17.242406+0000 mgr.y (mgr.34107) 432 : cluster [DBG] pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:18.709 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:18 smithi089 bash[82331]: cluster 2024-04-13T07:49:17.242406+0000 mgr.y (mgr.34107) 432 : cluster [DBG] pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:18.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:18 smithi203 bash[68060]: cluster 2024-04-13T07:49:17.242406+0000 mgr.y (mgr.34107) 432 : cluster [DBG] pgmap v368: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:19.140 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:49:19.141 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:49:19.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:19 smithi203 bash[68060]: audit 2024-04-13T07:49:19.145408+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/723293076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:19.778 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:19 smithi089 bash[82331]: audit 2024-04-13T07:49:19.145408+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/723293076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:19.779 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:19 smithi089 bash[84777]: audit 2024-04-13T07:49:19.145408+0000 mon.c (mon.2) 38 : audit [DBG] from='client.? 172.21.15.89:0/723293076' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:20.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:20 smithi203 bash[68060]: cluster 2024-04-13T07:49:19.243378+0000 mgr.y (mgr.34107) 433 : cluster [DBG] pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:20.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:20 smithi089 bash[82331]: cluster 2024-04-13T07:49:19.243378+0000 mgr.y (mgr.34107) 433 : cluster [DBG] pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:20.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:20 smithi089 bash[84777]: cluster 2024-04-13T07:49:19.243378+0000 mgr.y (mgr.34107) 433 : cluster [DBG] pgmap v369: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:22.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:22 smithi203 bash[68060]: cluster 2024-04-13T07:49:21.244118+0000 mgr.y (mgr.34107) 434 : cluster [DBG] pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:22.768 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:49:22 smithi203 bash[60252]: ts=2024-04-13T07:49:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:49:22.781 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:49:22.792 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:22 smithi089 bash[82331]: cluster 2024-04-13T07:49:21.244118+0000 mgr.y (mgr.34107) 434 : cluster [DBG] pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:22.792 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:22 smithi089 bash[84777]: cluster 2024-04-13T07:49:21.244118+0000 mgr.y (mgr.34107) 434 : cluster [DBG] pgmap v370: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:24.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:24 smithi203 bash[68060]: cluster 2024-04-13T07:49:23.245040+0000 mgr.y (mgr.34107) 435 : cluster [DBG] pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:24.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:24 smithi089 bash[82331]: cluster 2024-04-13T07:49:23.245040+0000 mgr.y (mgr.34107) 435 : cluster [DBG] pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:24.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:24 smithi089 bash[84777]: cluster 2024-04-13T07:49:23.245040+0000 mgr.y (mgr.34107) 435 : cluster [DBG] pgmap v371: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:25.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:25 smithi203 bash[68060]: audit 2024-04-13T07:49:24.923043+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:49:25.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:25 smithi089 bash[82331]: audit 2024-04-13T07:49:24.923043+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:49:25.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:25 smithi089 bash[84777]: audit 2024-04-13T07:49:24.923043+0000 mon.a (mon.0) 650 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:49:26.299 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:49:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:49:25] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:49:26.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:26 smithi203 bash[68060]: cluster 2024-04-13T07:49:25.245701+0000 mgr.y (mgr.34107) 436 : cluster [DBG] pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:26.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:26 smithi089 bash[84777]: cluster 2024-04-13T07:49:25.245701+0000 mgr.y (mgr.34107) 436 : cluster [DBG] pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:26.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:26 smithi089 bash[82331]: cluster 2024-04-13T07:49:25.245701+0000 mgr.y (mgr.34107) 436 : cluster [DBG] pgmap v372: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:27.515 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:49:27.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:27 smithi203 bash[68060]: audit 2024-04-13T07:49:27.082360+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:27.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:27 smithi089 bash[84777]: audit 2024-04-13T07:49:27.082360+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:27.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:27 smithi089 bash[82331]: audit 2024-04-13T07:49:27.082360+0000 mon.a (mon.0) 651 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:28.721 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:28 smithi089 bash[82331]: cluster 2024-04-13T07:49:27.246477+0000 mgr.y (mgr.34107) 437 : cluster [DBG] pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:28.722 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:28 smithi089 bash[84777]: cluster 2024-04-13T07:49:27.246477+0000 mgr.y (mgr.34107) 437 : cluster [DBG] pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:28.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:28 smithi203 bash[68060]: cluster 2024-04-13T07:49:27.246477+0000 mgr.y (mgr.34107) 437 : cluster [DBG] pgmap v373: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:30.391 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:49:30.391 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:49:30.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:30 smithi203 bash[68060]: cluster 2024-04-13T07:49:29.247044+0000 mgr.y (mgr.34107) 438 : cluster [DBG] pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:30.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:30 smithi203 bash[68060]: audit 2024-04-13T07:49:30.396468+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.89:0/1396126842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:30.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:30 smithi203 bash[68060]: audit 2024-04-13T07:49:30.414919+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:30.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:30 smithi203 bash[68060]: audit 2024-04-13T07:49:30.421599+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:30.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:30 smithi089 bash[82331]: cluster 2024-04-13T07:49:29.247044+0000 mgr.y (mgr.34107) 438 : cluster [DBG] pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:30.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:30 smithi089 bash[82331]: audit 2024-04-13T07:49:30.396468+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.89:0/1396126842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:30.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:30 smithi089 bash[82331]: audit 2024-04-13T07:49:30.414919+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:30.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:30 smithi089 bash[82331]: audit 2024-04-13T07:49:30.421599+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:30 smithi089 bash[84777]: cluster 2024-04-13T07:49:29.247044+0000 mgr.y (mgr.34107) 438 : cluster [DBG] pgmap v374: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:30 smithi089 bash[84777]: audit 2024-04-13T07:49:30.396468+0000 mon.a (mon.0) 652 : audit [DBG] from='client.? 172.21.15.89:0/1396126842' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:30 smithi089 bash[84777]: audit 2024-04-13T07:49:30.414919+0000 mon.a (mon.0) 653 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:30.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:30 smithi089 bash[84777]: audit 2024-04-13T07:49:30.421599+0000 mon.a (mon.0) 654 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:32 smithi203 bash[68060]: audit 2024-04-13T07:49:31.023235+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.268 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:32 smithi203 bash[68060]: audit 2024-04-13T07:49:31.034784+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:32 smithi203 bash[68060]: cluster 2024-04-13T07:49:31.247826+0000 mgr.y (mgr.34107) 439 : cluster [DBG] pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:32.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:32 smithi203 bash[68060]: audit 2024-04-13T07:49:31.660303+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:49:32.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:32 smithi203 bash[68060]: audit 2024-04-13T07:49:31.661801+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:49:32.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:32 smithi203 bash[68060]: audit 2024-04-13T07:49:31.672719+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[84777]: audit 2024-04-13T07:49:31.023235+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[84777]: audit 2024-04-13T07:49:31.034784+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[84777]: cluster 2024-04-13T07:49:31.247826+0000 mgr.y (mgr.34107) 439 : cluster [DBG] pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:32.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[84777]: audit 2024-04-13T07:49:31.660303+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:49:32.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[84777]: audit 2024-04-13T07:49:31.661801+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:49:32.300 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[84777]: audit 2024-04-13T07:49:31.672719+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[82331]: audit 2024-04-13T07:49:31.023235+0000 mon.a (mon.0) 655 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[82331]: audit 2024-04-13T07:49:31.034784+0000 mon.a (mon.0) 656 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[82331]: cluster 2024-04-13T07:49:31.247826+0000 mgr.y (mgr.34107) 439 : cluster [DBG] pgmap v375: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:32.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[82331]: audit 2024-04-13T07:49:31.660303+0000 mon.a (mon.0) 657 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:49:32.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[82331]: audit 2024-04-13T07:49:31.661801+0000 mon.a (mon.0) 658 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:49:32.301 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:32 smithi089 bash[82331]: audit 2024-04-13T07:49:31.672719+0000 mon.a (mon.0) 659 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:49:32.768 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:49:32 smithi203 bash[60252]: ts=2024-04-13T07:49:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:49:34.054 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:49:34.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:34 smithi089 bash[84777]: cluster 2024-04-13T07:49:33.248786+0000 mgr.y (mgr.34107) 440 : cluster [DBG] pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:34.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:34 smithi089 bash[82331]: cluster 2024-04-13T07:49:33.248786+0000 mgr.y (mgr.34107) 440 : cluster [DBG] pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:34.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:34 smithi203 bash[68060]: cluster 2024-04-13T07:49:33.248786+0000 mgr.y (mgr.34107) 440 : cluster [DBG] pgmap v376: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:36.300 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:49:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:49:35] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:49:36.770 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:36 smithi203 bash[68060]: cluster 2024-04-13T07:49:35.249476+0000 mgr.y (mgr.34107) 441 : cluster [DBG] pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:36.799 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:36 smithi089 bash[82331]: cluster 2024-04-13T07:49:35.249476+0000 mgr.y (mgr.34107) 441 : cluster [DBG] pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:36.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:36 smithi089 bash[84777]: cluster 2024-04-13T07:49:35.249476+0000 mgr.y (mgr.34107) 441 : cluster [DBG] pgmap v377: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:38.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:38 smithi203 bash[68060]: cluster 2024-04-13T07:49:37.250245+0000 mgr.y (mgr.34107) 442 : cluster [DBG] pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:38.791 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:49:38.799 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:38 smithi089 bash[84777]: cluster 2024-04-13T07:49:37.250245+0000 mgr.y (mgr.34107) 442 : cluster [DBG] pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:38.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:38 smithi089 bash[82331]: cluster 2024-04-13T07:49:37.250245+0000 mgr.y (mgr.34107) 442 : cluster [DBG] pgmap v378: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:40.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:40 smithi203 bash[68060]: cluster 2024-04-13T07:49:39.251192+0000 mgr.y (mgr.34107) 443 : cluster [DBG] pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:40.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:40 smithi089 bash[82331]: cluster 2024-04-13T07:49:39.251192+0000 mgr.y (mgr.34107) 443 : cluster [DBG] pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:40.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:40 smithi089 bash[84777]: cluster 2024-04-13T07:49:39.251192+0000 mgr.y (mgr.34107) 443 : cluster [DBG] pgmap v379: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:41.527 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:49:41.528 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:49:42.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:42 smithi089 bash[84777]: cluster 2024-04-13T07:49:41.251915+0000 mgr.y (mgr.34107) 444 : cluster [DBG] pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:42.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:42 smithi089 bash[84777]: audit 2024-04-13T07:49:41.532425+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.89:0/297265045' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:42.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:42 smithi089 bash[84777]: audit 2024-04-13T07:49:42.082730+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:42.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:42 smithi089 bash[82331]: cluster 2024-04-13T07:49:41.251915+0000 mgr.y (mgr.34107) 444 : cluster [DBG] pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:42.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:42 smithi089 bash[82331]: audit 2024-04-13T07:49:41.532425+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.89:0/297265045' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:42.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:42 smithi089 bash[82331]: audit 2024-04-13T07:49:42.082730+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:42.768 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:49:42 smithi203 bash[60252]: ts=2024-04-13T07:49:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:49:42.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:42 smithi203 bash[68060]: cluster 2024-04-13T07:49:41.251915+0000 mgr.y (mgr.34107) 444 : cluster [DBG] pgmap v380: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:42.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:42 smithi203 bash[68060]: audit 2024-04-13T07:49:41.532425+0000 mon.a (mon.0) 660 : audit [DBG] from='client.? 172.21.15.89:0/297265045' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:42.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:42 smithi203 bash[68060]: audit 2024-04-13T07:49:42.082730+0000 mon.a (mon.0) 661 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:44.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:44 smithi203 bash[68060]: cluster 2024-04-13T07:49:43.252824+0000 mgr.y (mgr.34107) 445 : cluster [DBG] pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:44.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:44 smithi089 bash[84777]: cluster 2024-04-13T07:49:43.252824+0000 mgr.y (mgr.34107) 445 : cluster [DBG] pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:44.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:44 smithi089 bash[82331]: cluster 2024-04-13T07:49:43.252824+0000 mgr.y (mgr.34107) 445 : cluster [DBG] pgmap v381: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:45.125 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:49:46.300 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:49:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:49:45] "GET /metrics HTTP/1.1" 200 34076 "" "Prometheus/2.43.0" 2024-04-13T07:49:46.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:46 smithi203 bash[68060]: cluster 2024-04-13T07:49:45.253526+0000 mgr.y (mgr.34107) 446 : cluster [DBG] pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:46.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:46 smithi089 bash[82331]: cluster 2024-04-13T07:49:45.253526+0000 mgr.y (mgr.34107) 446 : cluster [DBG] pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:46.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:46 smithi089 bash[84777]: cluster 2024-04-13T07:49:45.253526+0000 mgr.y (mgr.34107) 446 : cluster [DBG] pgmap v382: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:48.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:48 smithi203 bash[68060]: cluster 2024-04-13T07:49:47.254350+0000 mgr.y (mgr.34107) 447 : cluster [DBG] pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:48.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:48 smithi089 bash[84777]: cluster 2024-04-13T07:49:47.254350+0000 mgr.y (mgr.34107) 447 : cluster [DBG] pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:48.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:48 smithi089 bash[82331]: cluster 2024-04-13T07:49:47.254350+0000 mgr.y (mgr.34107) 447 : cluster [DBG] pgmap v383: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:49.871 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:49:50.628 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:50 smithi089 bash[82331]: cluster 2024-04-13T07:49:49.255312+0000 mgr.y (mgr.34107) 448 : cluster [DBG] pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:50.628 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:50 smithi089 bash[84777]: cluster 2024-04-13T07:49:49.255312+0000 mgr.y (mgr.34107) 448 : cluster [DBG] pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:50.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:50 smithi203 bash[68060]: cluster 2024-04-13T07:49:49.255312+0000 mgr.y (mgr.34107) 448 : cluster [DBG] pgmap v384: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:52.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:52 smithi089 bash[82331]: cluster 2024-04-13T07:49:51.256125+0000 mgr.y (mgr.34107) 449 : cluster [DBG] pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:52.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:52 smithi089 bash[84777]: cluster 2024-04-13T07:49:51.256125+0000 mgr.y (mgr.34107) 449 : cluster [DBG] pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:52.610 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:49:52.610 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:49:52.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:52 smithi203 bash[68060]: cluster 2024-04-13T07:49:51.256125+0000 mgr.y (mgr.34107) 449 : cluster [DBG] pgmap v385: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:52.769 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:49:52 smithi203 bash[60252]: ts=2024-04-13T07:49:52.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:49:53.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:53 smithi089 bash[82331]: audit 2024-04-13T07:49:52.614795+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.89:0/2221961741' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:53.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:53 smithi089 bash[84777]: audit 2024-04-13T07:49:52.614795+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.89:0/2221961741' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:53.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:53 smithi203 bash[68060]: audit 2024-04-13T07:49:52.614795+0000 mon.c (mon.2) 39 : audit [DBG] from='client.? 172.21.15.89:0/2221961741' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:49:54.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:54 smithi203 bash[68060]: cluster 2024-04-13T07:49:53.257068+0000 mgr.y (mgr.34107) 450 : cluster [DBG] pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:54.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:54 smithi089 bash[82331]: cluster 2024-04-13T07:49:53.257068+0000 mgr.y (mgr.34107) 450 : cluster [DBG] pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:54.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:54 smithi089 bash[84777]: cluster 2024-04-13T07:49:53.257068+0000 mgr.y (mgr.34107) 450 : cluster [DBG] pgmap v386: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:56.277 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:49:56.288 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:49:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:49:55] "GET /metrics HTTP/1.1" 200 34076 "" "Prometheus/2.43.0" 2024-04-13T07:49:56.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:56 smithi089 bash[82331]: cluster 2024-04-13T07:49:55.257834+0000 mgr.y (mgr.34107) 451 : cluster [DBG] pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:56.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:56 smithi089 bash[84777]: cluster 2024-04-13T07:49:55.257834+0000 mgr.y (mgr.34107) 451 : cluster [DBG] pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:56.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:56 smithi203 bash[68060]: cluster 2024-04-13T07:49:55.257834+0000 mgr.y (mgr.34107) 451 : cluster [DBG] pgmap v387: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:57.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:57 smithi203 bash[68060]: audit 2024-04-13T07:49:57.083200+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:57.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:57 smithi089 bash[82331]: audit 2024-04-13T07:49:57.083200+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:57.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:57 smithi089 bash[84777]: audit 2024-04-13T07:49:57.083200+0000 mon.a (mon.0) 662 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:49:58.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:49:58 smithi203 bash[68060]: cluster 2024-04-13T07:49:57.258624+0000 mgr.y (mgr.34107) 452 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:58.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:49:58 smithi089 bash[84777]: cluster 2024-04-13T07:49:57.258624+0000 mgr.y (mgr.34107) 452 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:49:58.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:49:58 smithi089 bash[82331]: cluster 2024-04-13T07:49:57.258624+0000 mgr.y (mgr.34107) 452 : cluster [DBG] pgmap v388: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:00.300 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:00 smithi089 bash[82331]: debug 2024-04-13T07:49:59.998+0000 7f9608649700 -1 log_channel(cluster) log [ERR] : overall HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:50:00.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:00 smithi203 bash[68060]: cluster 2024-04-13T07:49:59.259582+0000 mgr.y (mgr.34107) 453 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:00.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:00 smithi203 bash[68060]: cluster 2024-04-13T07:50:00.000123+0000 mon.a (mon.0) 663 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:50:00.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:00 smithi089 bash[82331]: cluster 2024-04-13T07:49:59.259582+0000 mgr.y (mgr.34107) 453 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:00.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:00 smithi089 bash[82331]: cluster 2024-04-13T07:50:00.000123+0000 mon.a (mon.0) 663 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:50:00.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:00 smithi089 bash[84777]: cluster 2024-04-13T07:49:59.259582+0000 mgr.y (mgr.34107) 453 : cluster [DBG] pgmap v389: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:00.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:00 smithi089 bash[84777]: cluster 2024-04-13T07:50:00.000123+0000 mon.a (mon.0) 663 : cluster [ERR] overall HEALTH_ERR 8 osds(s) are not reachable 2024-04-13T07:50:01.015 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:50:02.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:02 smithi089 bash[82331]: cluster 2024-04-13T07:50:01.260339+0000 mgr.y (mgr.34107) 454 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:02.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:02 smithi089 bash[84777]: cluster 2024-04-13T07:50:01.260339+0000 mgr.y (mgr.34107) 454 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:02.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:02 smithi203 bash[68060]: cluster 2024-04-13T07:50:01.260339+0000 mgr.y (mgr.34107) 454 : cluster [DBG] pgmap v390: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:02.769 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:50:02 smithi203 bash[60252]: ts=2024-04-13T07:50:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:50:03.923 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:50:03.923 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:50:04.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:04 smithi203 bash[68060]: cluster 2024-04-13T07:50:03.261215+0000 mgr.y (mgr.34107) 455 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:04.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:04 smithi203 bash[68060]: audit 2024-04-13T07:50:03.927476+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.89:0/2790239653' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:04.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:04 smithi089 bash[84777]: cluster 2024-04-13T07:50:03.261215+0000 mgr.y (mgr.34107) 455 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:04.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:04 smithi089 bash[84777]: audit 2024-04-13T07:50:03.927476+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.89:0/2790239653' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:04.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:04 smithi089 bash[82331]: cluster 2024-04-13T07:50:03.261215+0000 mgr.y (mgr.34107) 455 : cluster [DBG] pgmap v391: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:04.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:04 smithi089 bash[82331]: audit 2024-04-13T07:50:03.927476+0000 mon.c (mon.2) 40 : audit [DBG] from='client.? 172.21.15.89:0/2790239653' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:06.300 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:50:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:50:05] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:50:06.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:06 smithi203 bash[68060]: cluster 2024-04-13T07:50:05.261909+0000 mgr.y (mgr.34107) 456 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:06.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:06 smithi089 bash[84777]: cluster 2024-04-13T07:50:05.261909+0000 mgr.y (mgr.34107) 456 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:06.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:06 smithi089 bash[82331]: cluster 2024-04-13T07:50:05.261909+0000 mgr.y (mgr.34107) 456 : cluster [DBG] pgmap v392: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:07.525 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:50:08.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:08 smithi203 bash[68060]: cluster 2024-04-13T07:50:07.262695+0000 mgr.y (mgr.34107) 457 : cluster [DBG] pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:08.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:08 smithi089 bash[82331]: cluster 2024-04-13T07:50:07.262695+0000 mgr.y (mgr.34107) 457 : cluster [DBG] pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:08.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:08 smithi089 bash[84777]: cluster 2024-04-13T07:50:07.262695+0000 mgr.y (mgr.34107) 457 : cluster [DBG] pgmap v393: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:10.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:10 smithi203 bash[68060]: cluster 2024-04-13T07:50:09.263631+0000 mgr.y (mgr.34107) 458 : cluster [DBG] pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:10.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:10 smithi089 bash[82331]: cluster 2024-04-13T07:50:09.263631+0000 mgr.y (mgr.34107) 458 : cluster [DBG] pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:10.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:10 smithi089 bash[84777]: cluster 2024-04-13T07:50:09.263631+0000 mgr.y (mgr.34107) 458 : cluster [DBG] pgmap v394: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:12.270 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:50:12.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:12 smithi089 bash[84777]: cluster 2024-04-13T07:50:11.264391+0000 mgr.y (mgr.34107) 459 : cluster [DBG] pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:12.550 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:12 smithi089 bash[84777]: audit 2024-04-13T07:50:12.083740+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:12.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:12 smithi089 bash[82331]: cluster 2024-04-13T07:50:11.264391+0000 mgr.y (mgr.34107) 459 : cluster [DBG] pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:12.551 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:12 smithi089 bash[82331]: audit 2024-04-13T07:50:12.083740+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:12.770 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:12 smithi203 bash[68060]: cluster 2024-04-13T07:50:11.264391+0000 mgr.y (mgr.34107) 459 : cluster [DBG] pgmap v395: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:12.770 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:12 smithi203 bash[68060]: audit 2024-04-13T07:50:12.083740+0000 mon.a (mon.0) 664 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:12.770 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:50:12 smithi203 bash[60252]: ts=2024-04-13T07:50:12.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:50:14.759 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:14 smithi089 bash[84777]: cluster 2024-04-13T07:50:13.265334+0000 mgr.y (mgr.34107) 460 : cluster [DBG] pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:14.759 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:14 smithi089 bash[82331]: cluster 2024-04-13T07:50:13.265334+0000 mgr.y (mgr.34107) 460 : cluster [DBG] pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:14.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:14 smithi203 bash[68060]: cluster 2024-04-13T07:50:13.265334+0000 mgr.y (mgr.34107) 460 : cluster [DBG] pgmap v396: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:15.190 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:50:15.190 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:50:15.550 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:15 smithi089 bash[82331]: audit 2024-04-13T07:50:15.194568+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/3131794295' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:15.551 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:15 smithi089 bash[84777]: audit 2024-04-13T07:50:15.194568+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/3131794295' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:15.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:15 smithi203 bash[68060]: audit 2024-04-13T07:50:15.194568+0000 mon.c (mon.2) 41 : audit [DBG] from='client.? 172.21.15.89:0/3131794295' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:16.300 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:50:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:50:15] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:50:16.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:16 smithi203 bash[68060]: cluster 2024-04-13T07:50:15.266047+0000 mgr.y (mgr.34107) 461 : cluster [DBG] pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:16.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:16 smithi089 bash[82331]: cluster 2024-04-13T07:50:15.266047+0000 mgr.y (mgr.34107) 461 : cluster [DBG] pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:16.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:16 smithi089 bash[84777]: cluster 2024-04-13T07:50:15.266047+0000 mgr.y (mgr.34107) 461 : cluster [DBG] pgmap v397: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:18.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:18 smithi203 bash[68060]: cluster 2024-04-13T07:50:17.266793+0000 mgr.y (mgr.34107) 462 : cluster [DBG] pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:18.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:18 smithi089 bash[82331]: cluster 2024-04-13T07:50:17.266793+0000 mgr.y (mgr.34107) 462 : cluster [DBG] pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:18.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:18 smithi089 bash[84777]: cluster 2024-04-13T07:50:17.266793+0000 mgr.y (mgr.34107) 462 : cluster [DBG] pgmap v398: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:18.886 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:50:20.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:20 smithi203 bash[68060]: cluster 2024-04-13T07:50:19.267739+0000 mgr.y (mgr.34107) 463 : cluster [DBG] pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:20.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:20 smithi089 bash[82331]: cluster 2024-04-13T07:50:19.267739+0000 mgr.y (mgr.34107) 463 : cluster [DBG] pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:20.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:20 smithi089 bash[84777]: cluster 2024-04-13T07:50:19.267739+0000 mgr.y (mgr.34107) 463 : cluster [DBG] pgmap v399: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:22.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:22 smithi203 bash[68060]: cluster 2024-04-13T07:50:21.268443+0000 mgr.y (mgr.34107) 464 : cluster [DBG] pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:22.769 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:50:22 smithi203 bash[60252]: ts=2024-04-13T07:50:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:50:22.802 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:22 smithi089 bash[84777]: cluster 2024-04-13T07:50:21.268443+0000 mgr.y (mgr.34107) 464 : cluster [DBG] pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:22.802 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:22 smithi089 bash[82331]: cluster 2024-04-13T07:50:21.268443+0000 mgr.y (mgr.34107) 464 : cluster [DBG] pgmap v400: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:23.619 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:50:24.755 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:24 smithi089 bash[82331]: cluster 2024-04-13T07:50:23.269398+0000 mgr.y (mgr.34107) 465 : cluster [DBG] pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:24.755 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:24 smithi089 bash[84777]: cluster 2024-04-13T07:50:23.269398+0000 mgr.y (mgr.34107) 465 : cluster [DBG] pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:24.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:24 smithi203 bash[68060]: cluster 2024-04-13T07:50:23.269398+0000 mgr.y (mgr.34107) 465 : cluster [DBG] pgmap v401: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:26.300 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:50:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:50:25] "GET /metrics HTTP/1.1" 200 34074 "" "Prometheus/2.43.0" 2024-04-13T07:50:26.431 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:50:26.431 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:50:26.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:26 smithi203 bash[68060]: cluster 2024-04-13T07:50:25.270049+0000 mgr.y (mgr.34107) 466 : cluster [DBG] pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:26.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:26 smithi089 bash[82331]: cluster 2024-04-13T07:50:25.270049+0000 mgr.y (mgr.34107) 466 : cluster [DBG] pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:26.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:26 smithi089 bash[84777]: cluster 2024-04-13T07:50:25.270049+0000 mgr.y (mgr.34107) 466 : cluster [DBG] pgmap v402: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:27.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:27 smithi203 bash[68060]: audit 2024-04-13T07:50:26.435290+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.89:0/1001770515' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:27.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:27 smithi203 bash[68060]: audit 2024-04-13T07:50:27.084803+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:27.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:27 smithi089 bash[84777]: audit 2024-04-13T07:50:26.435290+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.89:0/1001770515' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:27.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:27 smithi089 bash[84777]: audit 2024-04-13T07:50:27.084803+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:27.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:27 smithi089 bash[82331]: audit 2024-04-13T07:50:26.435290+0000 mon.a (mon.0) 665 : audit [DBG] from='client.? 172.21.15.89:0/1001770515' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:27.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:27 smithi089 bash[82331]: audit 2024-04-13T07:50:27.084803+0000 mon.a (mon.0) 666 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:28.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:28 smithi203 bash[68060]: cluster 2024-04-13T07:50:27.270834+0000 mgr.y (mgr.34107) 467 : cluster [DBG] pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:28.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:28 smithi089 bash[82331]: cluster 2024-04-13T07:50:27.270834+0000 mgr.y (mgr.34107) 467 : cluster [DBG] pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:28.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:28 smithi089 bash[84777]: cluster 2024-04-13T07:50:27.270834+0000 mgr.y (mgr.34107) 467 : cluster [DBG] pgmap v403: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:30.102 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:50:30.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:30 smithi203 bash[68060]: cluster 2024-04-13T07:50:29.271719+0000 mgr.y (mgr.34107) 468 : cluster [DBG] pgmap v404: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:30.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:30 smithi089 bash[82331]: cluster 2024-04-13T07:50:29.271719+0000 mgr.y (mgr.34107) 468 : cluster [DBG] pgmap v404: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:30.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:30 smithi089 bash[84777]: cluster 2024-04-13T07:50:29.271719+0000 mgr.y (mgr.34107) 468 : cluster [DBG] pgmap v404: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:32.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:32 smithi203 bash[68060]: cluster 2024-04-13T07:50:31.272426+0000 mgr.y (mgr.34107) 469 : cluster [DBG] pgmap v405: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:32.769 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:32 smithi203 bash[68060]: audit 2024-04-13T07:50:31.762564+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:50:32.769 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:50:32 smithi203 bash[60252]: ts=2024-04-13T07:50:32.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:50:32.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:32 smithi089 bash[82331]: cluster 2024-04-13T07:50:31.272426+0000 mgr.y (mgr.34107) 469 : cluster [DBG] pgmap v405: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:32.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:32 smithi089 bash[82331]: audit 2024-04-13T07:50:31.762564+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:50:32.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:32 smithi089 bash[84777]: cluster 2024-04-13T07:50:31.272426+0000 mgr.y (mgr.34107) 469 : cluster [DBG] pgmap v405: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:32.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:32 smithi089 bash[84777]: audit 2024-04-13T07:50:31.762564+0000 mon.a (mon.0) 667 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2024-04-13T07:50:34.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:34 smithi203 bash[68060]: cluster 2024-04-13T07:50:33.273412+0000 mgr.y (mgr.34107) 470 : cluster [DBG] pgmap v406: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:34.800 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:34 smithi089 bash[84777]: cluster 2024-04-13T07:50:33.273412+0000 mgr.y (mgr.34107) 470 : cluster [DBG] pgmap v406: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:34.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:34 smithi089 bash[82331]: cluster 2024-04-13T07:50:33.273412+0000 mgr.y (mgr.34107) 470 : cluster [DBG] pgmap v406: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:34.827 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:50:36.300 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:50:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:50:35] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:50:36.768 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:36 smithi203 bash[68060]: cluster 2024-04-13T07:50:35.274149+0000 mgr.y (mgr.34107) 471 : cluster [DBG] pgmap v407: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:36.800 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:36 smithi089 bash[82331]: cluster 2024-04-13T07:50:35.274149+0000 mgr.y (mgr.34107) 471 : cluster [DBG] pgmap v407: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:36.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:36 smithi089 bash[84777]: cluster 2024-04-13T07:50:35.274149+0000 mgr.y (mgr.34107) 471 : cluster [DBG] pgmap v407: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:37.340 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:50:37.340 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:50:37.997 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:37 smithi089 bash[82331]: audit 2024-04-13T07:50:36.652763+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:37.997 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:37 smithi089 bash[82331]: audit 2024-04-13T07:50:36.658736+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:37.997 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:37 smithi089 bash[82331]: audit 2024-04-13T07:50:37.344516+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.89:0/1102381764' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:37.998 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:37 smithi089 bash[84777]: audit 2024-04-13T07:50:36.652763+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:37.998 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:37 smithi089 bash[84777]: audit 2024-04-13T07:50:36.658736+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:37.998 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:37 smithi089 bash[84777]: audit 2024-04-13T07:50:37.344516+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.89:0/1102381764' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:38.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:37 smithi203 bash[68060]: audit 2024-04-13T07:50:36.652763+0000 mon.a (mon.0) 668 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:38.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:37 smithi203 bash[68060]: audit 2024-04-13T07:50:36.658736+0000 mon.a (mon.0) 669 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:38.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:37 smithi203 bash[68060]: audit 2024-04-13T07:50:37.344516+0000 mon.a (mon.0) 670 : audit [DBG] from='client.? 172.21.15.89:0/1102381764' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:39.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:38 smithi203 bash[68060]: cluster 2024-04-13T07:50:37.274858+0000 mgr.y (mgr.34107) 472 : cluster [DBG] pgmap v408: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:39.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:38 smithi203 bash[68060]: audit 2024-04-13T07:50:37.866149+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:39.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:38 smithi203 bash[68060]: audit 2024-04-13T07:50:37.877304+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:39.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:38 smithi203 bash[68060]: audit 2024-04-13T07:50:38.558384+0000 mon.a (mon.0) 673 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:50:39.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:38 smithi203 bash[68060]: audit 2024-04-13T07:50:38.560394+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:50:39.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:38 smithi203 bash[68060]: audit 2024-04-13T07:50:38.570906+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[82331]: cluster 2024-04-13T07:50:37.274858+0000 mgr.y (mgr.34107) 472 : cluster [DBG] pgmap v408: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[82331]: audit 2024-04-13T07:50:37.866149+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[82331]: audit 2024-04-13T07:50:37.877304+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[82331]: audit 2024-04-13T07:50:38.558384+0000 mon.a (mon.0) 673 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[82331]: audit 2024-04-13T07:50:38.560394+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[82331]: audit 2024-04-13T07:50:38.570906+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[84777]: cluster 2024-04-13T07:50:37.274858+0000 mgr.y (mgr.34107) 472 : cluster [DBG] pgmap v408: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[84777]: audit 2024-04-13T07:50:37.866149+0000 mon.a (mon.0) 671 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[84777]: audit 2024-04-13T07:50:37.877304+0000 mon.a (mon.0) 672 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[84777]: audit 2024-04-13T07:50:38.558384+0000 mon.a (mon.0) 673 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2024-04-13T07:50:39.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[84777]: audit 2024-04-13T07:50:38.560394+0000 mon.a (mon.0) 674 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2024-04-13T07:50:39.052 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:38 smithi089 bash[84777]: audit 2024-04-13T07:50:38.570906+0000 mon.a (mon.0) 675 : audit [INF] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' 2024-04-13T07:50:40.999 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:50:41.011 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:40 smithi089 bash[84777]: cluster 2024-04-13T07:50:39.275904+0000 mgr.y (mgr.34107) 473 : cluster [DBG] pgmap v409: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:41.011 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:40 smithi089 bash[82331]: cluster 2024-04-13T07:50:39.275904+0000 mgr.y (mgr.34107) 473 : cluster [DBG] pgmap v409: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:41.018 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:40 smithi203 bash[68060]: cluster 2024-04-13T07:50:39.275904+0000 mgr.y (mgr.34107) 473 : cluster [DBG] pgmap v409: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:42.670 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:50:42 smithi203 bash[60252]: ts=2024-04-13T07:50:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:50:43.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:42 smithi203 bash[68060]: cluster 2024-04-13T07:50:41.276702+0000 mgr.y (mgr.34107) 474 : cluster [DBG] pgmap v410: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:43.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:42 smithi203 bash[68060]: audit 2024-04-13T07:50:42.085122+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:43.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:42 smithi089 bash[84777]: cluster 2024-04-13T07:50:41.276702+0000 mgr.y (mgr.34107) 474 : cluster [DBG] pgmap v410: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:43.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:42 smithi089 bash[84777]: audit 2024-04-13T07:50:42.085122+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:43.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:42 smithi089 bash[82331]: cluster 2024-04-13T07:50:41.276702+0000 mgr.y (mgr.34107) 474 : cluster [DBG] pgmap v410: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:43.052 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:42 smithi089 bash[82331]: audit 2024-04-13T07:50:42.085122+0000 mon.a (mon.0) 676 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:45.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:44 smithi203 bash[68060]: cluster 2024-04-13T07:50:43.277628+0000 mgr.y (mgr.34107) 475 : cluster [DBG] pgmap v411: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:45.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:44 smithi089 bash[82331]: cluster 2024-04-13T07:50:43.277628+0000 mgr.y (mgr.34107) 475 : cluster [DBG] pgmap v411: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:45.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:44 smithi089 bash[84777]: cluster 2024-04-13T07:50:43.277628+0000 mgr.y (mgr.34107) 475 : cluster [DBG] pgmap v411: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:45.727 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:50:46.050 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:50:45 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:50:45] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:50:46.885 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:46 smithi089 bash[82331]: cluster 2024-04-13T07:50:45.278336+0000 mgr.y (mgr.34107) 476 : cluster [DBG] pgmap v412: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:46.885 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:46 smithi089 bash[84777]: cluster 2024-04-13T07:50:45.278336+0000 mgr.y (mgr.34107) 476 : cluster [DBG] pgmap v412: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:47.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:46 smithi203 bash[68060]: cluster 2024-04-13T07:50:45.278336+0000 mgr.y (mgr.34107) 476 : cluster [DBG] pgmap v412: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:48.506 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:50:48.507 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:50:48.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:48 smithi089 bash[82331]: cluster 2024-04-13T07:50:47.279144+0000 mgr.y (mgr.34107) 477 : cluster [DBG] pgmap v413: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:48.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:48 smithi089 bash[82331]: audit 2024-04-13T07:50:48.510774+0000 mon.a (mon.0) 677 : audit [DBG] from='client.? 172.21.15.89:0/2130248245' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:48.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:48 smithi089 bash[84777]: cluster 2024-04-13T07:50:47.279144+0000 mgr.y (mgr.34107) 477 : cluster [DBG] pgmap v413: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:48.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:48 smithi089 bash[84777]: audit 2024-04-13T07:50:48.510774+0000 mon.a (mon.0) 677 : audit [DBG] from='client.? 172.21.15.89:0/2130248245' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:49.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:48 smithi203 bash[68060]: cluster 2024-04-13T07:50:47.279144+0000 mgr.y (mgr.34107) 477 : cluster [DBG] pgmap v413: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:49.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:48 smithi203 bash[68060]: audit 2024-04-13T07:50:48.510774+0000 mon.a (mon.0) 677 : audit [DBG] from='client.? 172.21.15.89:0/2130248245' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:50:51.022 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:50 smithi203 bash[68060]: cluster 2024-04-13T07:50:49.279805+0000 mgr.y (mgr.34107) 478 : cluster [DBG] pgmap v414: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:51.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:50 smithi089 bash[82331]: cluster 2024-04-13T07:50:49.279805+0000 mgr.y (mgr.34107) 478 : cluster [DBG] pgmap v414: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:51.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:50 smithi089 bash[84777]: cluster 2024-04-13T07:50:49.279805+0000 mgr.y (mgr.34107) 478 : cluster [DBG] pgmap v414: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:52.230 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:50:52.692 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:50:52 smithi203 bash[60252]: ts=2024-04-13T07:50:52.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:50:53.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:52 smithi203 bash[68060]: cluster 2024-04-13T07:50:51.280615+0000 mgr.y (mgr.34107) 479 : cluster [DBG] pgmap v415: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:53.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:52 smithi089 bash[82331]: cluster 2024-04-13T07:50:51.280615+0000 mgr.y (mgr.34107) 479 : cluster [DBG] pgmap v415: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:53.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:52 smithi089 bash[84777]: cluster 2024-04-13T07:50:51.280615+0000 mgr.y (mgr.34107) 479 : cluster [DBG] pgmap v415: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:55.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:54 smithi203 bash[68060]: cluster 2024-04-13T07:50:53.281532+0000 mgr.y (mgr.34107) 480 : cluster [DBG] pgmap v416: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:55.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:54 smithi089 bash[84777]: cluster 2024-04-13T07:50:53.281532+0000 mgr.y (mgr.34107) 480 : cluster [DBG] pgmap v416: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:55.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:54 smithi089 bash[82331]: cluster 2024-04-13T07:50:53.281532+0000 mgr.y (mgr.34107) 480 : cluster [DBG] pgmap v416: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:56.301 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:50:55 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:50:55] "GET /metrics HTTP/1.1" 200 34072 "" "Prometheus/2.43.0" 2024-04-13T07:50:56.702 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:50:56 smithi203 bash[49777]: logger=cleanup t=2024-04-13T07:50:56.314591207Z level=info msg="Completed cleanup jobs" duration=48.135598ms 2024-04-13T07:50:56.972 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:50:56.992 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:56 smithi089 bash[82331]: cluster 2024-04-13T07:50:55.282282+0000 mgr.y (mgr.34107) 481 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:56.992 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:56 smithi089 bash[84777]: cluster 2024-04-13T07:50:55.282282+0000 mgr.y (mgr.34107) 481 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:57.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:56 smithi203 bash[68060]: cluster 2024-04-13T07:50:55.282282+0000 mgr.y (mgr.34107) 481 : cluster [DBG] pgmap v417: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:57.967 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:57 smithi089 bash[82331]: audit 2024-04-13T07:50:57.085893+0000 mon.a (mon.0) 678 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:57.967 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:57 smithi089 bash[84777]: audit 2024-04-13T07:50:57.085893+0000 mon.a (mon.0) 678 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:58.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:57 smithi203 bash[68060]: audit 2024-04-13T07:50:57.085893+0000 mon.a (mon.0) 678 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:50:59.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:50:58 smithi203 bash[68060]: cluster 2024-04-13T07:50:57.283110+0000 mgr.y (mgr.34107) 482 : cluster [DBG] pgmap v418: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:59.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:50:58 smithi089 bash[82331]: cluster 2024-04-13T07:50:57.283110+0000 mgr.y (mgr.34107) 482 : cluster [DBG] pgmap v418: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:59.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:50:58 smithi089 bash[84777]: cluster 2024-04-13T07:50:57.283110+0000 mgr.y (mgr.34107) 482 : cluster [DBG] pgmap v418: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:50:59.870 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:50:59.870 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:51:00.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:00 smithi089 bash[84777]: cluster 2024-04-13T07:50:59.284024+0000 mgr.y (mgr.34107) 483 : cluster [DBG] pgmap v419: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:00.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:00 smithi089 bash[84777]: audit 2024-04-13T07:50:59.874006+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/641659010' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:00.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:00 smithi089 bash[82331]: cluster 2024-04-13T07:50:59.284024+0000 mgr.y (mgr.34107) 483 : cluster [DBG] pgmap v419: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:00.801 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:00 smithi089 bash[82331]: audit 2024-04-13T07:50:59.874006+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/641659010' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:01.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:00 smithi203 bash[68060]: cluster 2024-04-13T07:50:59.284024+0000 mgr.y (mgr.34107) 483 : cluster [DBG] pgmap v419: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:01.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:00 smithi203 bash[68060]: audit 2024-04-13T07:50:59.874006+0000 mon.c (mon.2) 42 : audit [DBG] from='client.? 172.21.15.89:0/641659010' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:02.722 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:51:02 smithi203 bash[60252]: ts=2024-04-13T07:51:02.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:51:03.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:02 smithi203 bash[68060]: cluster 2024-04-13T07:51:01.284779+0000 mgr.y (mgr.34107) 484 : cluster [DBG] pgmap v420: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:03.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:02 smithi089 bash[84777]: cluster 2024-04-13T07:51:01.284779+0000 mgr.y (mgr.34107) 484 : cluster [DBG] pgmap v420: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:03.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:02 smithi089 bash[82331]: cluster 2024-04-13T07:51:01.284779+0000 mgr.y (mgr.34107) 484 : cluster [DBG] pgmap v420: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:03.486 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:51:05.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:04 smithi203 bash[68060]: cluster 2024-04-13T07:51:03.285692+0000 mgr.y (mgr.34107) 485 : cluster [DBG] pgmap v421: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:05.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:04 smithi089 bash[82331]: cluster 2024-04-13T07:51:03.285692+0000 mgr.y (mgr.34107) 485 : cluster [DBG] pgmap v421: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:05.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:04 smithi089 bash[84777]: cluster 2024-04-13T07:51:03.285692+0000 mgr.y (mgr.34107) 485 : cluster [DBG] pgmap v421: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:06.301 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:05 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:51:05] "GET /metrics HTTP/1.1" 200 34071 "" "Prometheus/2.43.0" 2024-04-13T07:51:07.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:06 smithi203 bash[68060]: cluster 2024-04-13T07:51:05.286453+0000 mgr.y (mgr.34107) 486 : cluster [DBG] pgmap v422: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:07.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:06 smithi089 bash[84777]: cluster 2024-04-13T07:51:05.286453+0000 mgr.y (mgr.34107) 486 : cluster [DBG] pgmap v422: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:07.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:06 smithi089 bash[82331]: cluster 2024-04-13T07:51:05.286453+0000 mgr.y (mgr.34107) 486 : cluster [DBG] pgmap v422: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:08.221 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:51:09.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:08 smithi203 bash[68060]: cluster 2024-04-13T07:51:07.287388+0000 mgr.y (mgr.34107) 487 : cluster [DBG] pgmap v423: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:09.034 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:08 smithi089 bash[84777]: cluster 2024-04-13T07:51:07.287388+0000 mgr.y (mgr.34107) 487 : cluster [DBG] pgmap v423: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:09.034 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:08 smithi089 bash[82331]: cluster 2024-04-13T07:51:07.287388+0000 mgr.y (mgr.34107) 487 : cluster [DBG] pgmap v423: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:11.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:10 smithi203 bash[68060]: cluster 2024-04-13T07:51:09.288078+0000 mgr.y (mgr.34107) 488 : cluster [DBG] pgmap v424: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:11.050 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:10 smithi089 bash[82331]: cluster 2024-04-13T07:51:09.288078+0000 mgr.y (mgr.34107) 488 : cluster [DBG] pgmap v424: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:11.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:10 smithi089 bash[84777]: cluster 2024-04-13T07:51:09.288078+0000 mgr.y (mgr.34107) 488 : cluster [DBG] pgmap v424: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:11.104 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:51:11.104 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:51:12.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:11 smithi203 bash[68060]: audit 2024-04-13T07:51:11.108485+0000 mon.a (mon.0) 679 : audit [DBG] from='client.? 172.21.15.89:0/2818092518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:12.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:11 smithi089 bash[84777]: audit 2024-04-13T07:51:11.108485+0000 mon.a (mon.0) 679 : audit [DBG] from='client.? 172.21.15.89:0/2818092518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:12.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:11 smithi089 bash[82331]: audit 2024-04-13T07:51:11.108485+0000 mon.a (mon.0) 679 : audit [DBG] from='client.? 172.21.15.89:0/2818092518' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:12.757 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:51:12 smithi203 bash[60252]: ts=2024-04-13T07:51:12.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:51:13.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:12 smithi203 bash[68060]: cluster 2024-04-13T07:51:11.288830+0000 mgr.y (mgr.34107) 489 : cluster [DBG] pgmap v425: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:13.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:12 smithi203 bash[68060]: audit 2024-04-13T07:51:12.086561+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:51:13.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:12 smithi089 bash[82331]: cluster 2024-04-13T07:51:11.288830+0000 mgr.y (mgr.34107) 489 : cluster [DBG] pgmap v425: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:13.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:12 smithi089 bash[82331]: audit 2024-04-13T07:51:12.086561+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:51:13.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:12 smithi089 bash[84777]: cluster 2024-04-13T07:51:11.288830+0000 mgr.y (mgr.34107) 489 : cluster [DBG] pgmap v425: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:13.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:12 smithi089 bash[84777]: audit 2024-04-13T07:51:12.086561+0000 mon.a (mon.0) 680 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:51:14.770 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:51:15.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:14 smithi203 bash[68060]: cluster 2024-04-13T07:51:13.289788+0000 mgr.y (mgr.34107) 490 : cluster [DBG] pgmap v426: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:15.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:14 smithi089 bash[84777]: cluster 2024-04-13T07:51:13.289788+0000 mgr.y (mgr.34107) 490 : cluster [DBG] pgmap v426: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:15.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:14 smithi089 bash[82331]: cluster 2024-04-13T07:51:13.289788+0000 mgr.y (mgr.34107) 490 : cluster [DBG] pgmap v426: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:16.301 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:15 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:51:15] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.43.0" 2024-04-13T07:51:17.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:16 smithi203 bash[68060]: cluster 2024-04-13T07:51:15.290525+0000 mgr.y (mgr.34107) 491 : cluster [DBG] pgmap v427: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:17.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:16 smithi089 bash[82331]: cluster 2024-04-13T07:51:15.290525+0000 mgr.y (mgr.34107) 491 : cluster [DBG] pgmap v427: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:17.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:16 smithi089 bash[84777]: cluster 2024-04-13T07:51:15.290525+0000 mgr.y (mgr.34107) 491 : cluster [DBG] pgmap v427: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:19.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:18 smithi089 bash[82331]: cluster 2024-04-13T07:51:17.291333+0000 mgr.y (mgr.34107) 492 : cluster [DBG] pgmap v428: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:19.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:18 smithi089 bash[84777]: cluster 2024-04-13T07:51:17.291333+0000 mgr.y (mgr.34107) 492 : cluster [DBG] pgmap v428: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:19.270 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:18 smithi203 bash[68060]: cluster 2024-04-13T07:51:17.291333+0000 mgr.y (mgr.34107) 492 : cluster [DBG] pgmap v428: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:19.510 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:51:21.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:20 smithi089 bash[84777]: cluster 2024-04-13T07:51:19.292287+0000 mgr.y (mgr.34107) 493 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:21.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:20 smithi089 bash[82331]: cluster 2024-04-13T07:51:19.292287+0000 mgr.y (mgr.34107) 493 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:21.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:20 smithi203 bash[68060]: cluster 2024-04-13T07:51:19.292287+0000 mgr.y (mgr.34107) 493 : cluster [DBG] pgmap v429: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:22.316 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:51:22.316 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:51:22.769 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:51:22 smithi203 bash[60252]: ts=2024-04-13T07:51:22.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:51:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:22 smithi089 bash[82331]: cluster 2024-04-13T07:51:21.293045+0000 mgr.y (mgr.34107) 494 : cluster [DBG] pgmap v430: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:23.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:22 smithi089 bash[82331]: audit 2024-04-13T07:51:22.319577+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3488106488' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:23.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:22 smithi089 bash[84777]: cluster 2024-04-13T07:51:21.293045+0000 mgr.y (mgr.34107) 494 : cluster [DBG] pgmap v430: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:23.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:22 smithi089 bash[84777]: audit 2024-04-13T07:51:22.319577+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3488106488' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:23.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:22 smithi203 bash[68060]: cluster 2024-04-13T07:51:21.293045+0000 mgr.y (mgr.34107) 494 : cluster [DBG] pgmap v430: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:23.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:22 smithi203 bash[68060]: audit 2024-04-13T07:51:22.319577+0000 mon.c (mon.2) 43 : audit [DBG] from='client.? 172.21.15.89:0/3488106488' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:25.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:24 smithi089 bash[84777]: cluster 2024-04-13T07:51:23.294021+0000 mgr.y (mgr.34107) 495 : cluster [DBG] pgmap v431: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:25.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:24 smithi089 bash[82331]: cluster 2024-04-13T07:51:23.294021+0000 mgr.y (mgr.34107) 495 : cluster [DBG] pgmap v431: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:25.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:24 smithi203 bash[68060]: cluster 2024-04-13T07:51:23.294021+0000 mgr.y (mgr.34107) 495 : cluster [DBG] pgmap v431: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:25.947 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:reef shell --fsid ec50a262-f965-11ee-bc8d-c7b262605968 -- ceph health --format=json 2024-04-13T07:51:26.301 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:25 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:51:25] "GET /metrics HTTP/1.1" 200 34075 "" "Prometheus/2.43.0" 2024-04-13T07:51:27.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:26 smithi089 bash[82331]: cluster 2024-04-13T07:51:25.294761+0000 mgr.y (mgr.34107) 496 : cluster [DBG] pgmap v432: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:27.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:26 smithi089 bash[84777]: cluster 2024-04-13T07:51:25.294761+0000 mgr.y (mgr.34107) 496 : cluster [DBG] pgmap v432: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:27.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:26 smithi203 bash[68060]: cluster 2024-04-13T07:51:25.294761+0000 mgr.y (mgr.34107) 496 : cluster [DBG] pgmap v432: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:28.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:27 smithi089 bash[84777]: audit 2024-04-13T07:51:27.086453+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:51:28.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:27 smithi089 bash[82331]: audit 2024-04-13T07:51:27.086453+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:51:28.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:27 smithi203 bash[68060]: audit 2024-04-13T07:51:27.086453+0000 mon.a (mon.0) 681 : audit [DBG] from='mgr.34107 172.21.15.89:0/607196070' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2024-04-13T07:51:29.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:28 smithi089 bash[84777]: cluster 2024-04-13T07:51:27.295595+0000 mgr.y (mgr.34107) 497 : cluster [DBG] pgmap v433: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:29.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:28 smithi089 bash[82331]: cluster 2024-04-13T07:51:27.295595+0000 mgr.y (mgr.34107) 497 : cluster [DBG] pgmap v433: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:29.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:28 smithi203 bash[68060]: cluster 2024-04-13T07:51:27.295595+0000 mgr.y (mgr.34107) 497 : cluster [DBG] pgmap v433: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:30.694 INFO:teuthology.orchestra.run.smithi089.stderr:Inferring config /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/mon.a/config 2024-04-13T07:51:31.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:30 smithi089 bash[84777]: cluster 2024-04-13T07:51:29.296528+0000 mgr.y (mgr.34107) 498 : cluster [DBG] pgmap v434: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:31.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:30 smithi089 bash[82331]: cluster 2024-04-13T07:51:29.296528+0000 mgr.y (mgr.34107) 498 : cluster [DBG] pgmap v434: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:31.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:30 smithi203 bash[68060]: cluster 2024-04-13T07:51:29.296528+0000 mgr.y (mgr.34107) 498 : cluster [DBG] pgmap v434: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:32.769 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:51:32 smithi203 bash[60252]: ts=2024-04-13T07:51:32.314Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:51:33.104 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:32 smithi089 bash[82331]: cluster 2024-04-13T07:51:31.297240+0000 mgr.y (mgr.34107) 499 : cluster [DBG] pgmap v435: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:33.105 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:32 smithi089 bash[84777]: cluster 2024-04-13T07:51:31.297240+0000 mgr.y (mgr.34107) 499 : cluster [DBG] pgmap v435: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:33.269 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:32 smithi203 bash[68060]: cluster 2024-04-13T07:51:31.297240+0000 mgr.y (mgr.34107) 499 : cluster [DBG] pgmap v435: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:33.551 INFO:teuthology.orchestra.run.smithi089.stdout: 2024-04-13T07:51:33.552 INFO:teuthology.orchestra.run.smithi089.stdout:{"status":"HEALTH_ERR","checks":{"OSD_UNREACHABLE":{"severity":"HEALTH_ERR","summary":{"message":"8 osds(s) are not reachable","count":8},"muted":false}},"mutes":[]} 2024-04-13T07:51:34.051 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:33 smithi089 bash[82331]: audit 2024-04-13T07:51:33.555372+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/185891877' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:34.051 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:33 smithi089 bash[84777]: audit 2024-04-13T07:51:33.555372+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/185891877' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:34.227 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_c2d8c3155d854ee204cb7c7261232571229d1d54/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_c2d8c3155d854ee204cb7c7261232571229d1d54/qa/tasks/ceph_manager.py", line 3251, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-13T07:51:38.201 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=ddee5c7d8976497daf3b06d0f67fa21e Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_6c637841c215537a4502385240412f1966e0faab/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_ceph_c2d8c3155d854ee204cb7c7261232571229d1d54/qa/tasks/ceph.py", line 1478, in healthy manager.wait_until_healthy(timeout=300) File "/home/teuthworker/src/git.ceph.com_ceph_c2d8c3155d854ee204cb7c7261232571229d1d54/qa/tasks/ceph_manager.py", line 3251, in wait_until_healthy assert time.time() - start < timeout, \ AssertionError: timeout expired in wait_until_healthy 2024-04-13T07:51:38.204 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2024-04-13T07:51:38.213 INFO:tasks.cephadm:Teardown begin 2024-04-13T07:51:38.214 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-13T07:51:38.218 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:34 smithi089 bash[82331]: cluster 2024-04-13T07:51:33.297784+0000 mgr.y (mgr.34107) 500 : cluster [DBG] pgmap v436: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:38.218 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:34 smithi089 bash[84777]: cluster 2024-04-13T07:51:33.297784+0000 mgr.y (mgr.34107) 500 : cluster [DBG] pgmap v436: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:38.219 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:33 smithi203 bash[68060]: audit 2024-04-13T07:51:33.555372+0000 mon.c (mon.2) 44 : audit [DBG] from='client.? 172.21.15.89:0/185891877' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2024-04-13T07:51:38.219 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:34 smithi203 bash[68060]: cluster 2024-04-13T07:51:33.297784+0000 mgr.y (mgr.34107) 500 : cluster [DBG] pgmap v436: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:38.219 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:36 smithi203 bash[68060]: cluster 2024-04-13T07:51:35.298475+0000 mgr.y (mgr.34107) 501 : cluster [DBG] pgmap v437: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:38.219 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:35 smithi089 bash[74725]: ::ffff:172.21.15.203 - - [13/Apr/2024:07:51:35] "GET /metrics HTTP/1.1" 200 34073 "" "Prometheus/2.43.0" 2024-04-13T07:51:38.220 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:36 smithi089 bash[82331]: cluster 2024-04-13T07:51:35.298475+0000 mgr.y (mgr.34107) 501 : cluster [DBG] pgmap v437: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:38.220 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:36 smithi089 bash[84777]: cluster 2024-04-13T07:51:35.298475+0000 mgr.y (mgr.34107) 501 : cluster [DBG] pgmap v437: 73 pgs: 73 active+clean; 4.6 MiB data, 253 MiB used, 715 GiB / 715 GiB avail 2024-04-13T07:51:38.235 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-13T07:51:38.250 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2024-04-13T07:51:38.250 DEBUG:teuthology.orchestra.run.smithi089:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-13T07:51:38.283 DEBUG:teuthology.orchestra.run.smithi203:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2024-04-13T07:51:38.297 INFO:tasks.cephadm:Stopping all daemons... 2024-04-13T07:51:38.297 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2024-04-13T07:51:38.297 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a 2024-04-13T07:51:38.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:38 smithi089 systemd[1]: Stopping Ceph mon.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:51:38.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:38 smithi089 bash[82331]: debug 2024-04-13T07:51:38.471+0000 7f960b64f700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-13T07:51:38.487 INFO:journalctl@ceph.mon.a.smithi089.stdout:Apr 13 07:51:38 smithi089 bash[82331]: debug 2024-04-13T07:51:38.471+0000 7f960b64f700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2024-04-13T07:51:38.603 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:38 smithi089 bash[74725]: [13/Apr/2024:07:51:38] ENGINE Bus STOPPING 2024-04-13T07:51:38.603 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:38 smithi089 bash[74725]: [13/Apr/2024:07:51:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2024-04-13T07:51:38.603 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:38 smithi089 bash[74725]: [13/Apr/2024:07:51:38] ENGINE Bus STOPPED 2024-04-13T07:51:38.603 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:38 smithi089 bash[74725]: [13/Apr/2024:07:51:38] ENGINE Bus STARTING 2024-04-13T07:51:38.859 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:38 smithi089 bash[74725]: [13/Apr/2024:07:51:38] ENGINE Serving on http://:::9283 2024-04-13T07:51:38.859 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:38 smithi089 bash[74725]: [13/Apr/2024:07:51:38] ENGINE Bus STARTED 2024-04-13T07:51:39.508 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.a.service' 2024-04-13T07:51:39.541 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:51:39.542 INFO:tasks.cephadm.mon.a:Stopped mon.a 2024-04-13T07:51:39.542 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2024-04-13T07:51:39.542 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.c 2024-04-13T07:51:39.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:39 smithi089 systemd[1]: Stopping Ceph mon.c for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:51:39.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:39 smithi089 bash[84777]: debug 2024-04-13T07:51:39.719+0000 7f6915a61700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-13T07:51:39.801 INFO:journalctl@ceph.mon.c.smithi089.stdout:Apr 13 07:51:39 smithi089 bash[84777]: debug 2024-04-13T07:51:39.719+0000 7f6915a61700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2024-04-13T07:51:40.696 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.c.service' 2024-04-13T07:51:40.721 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:51:40.722 INFO:tasks.cephadm.mon.b:Stopped mon.c 2024-04-13T07:51:40.722 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2024-04-13T07:51:40.722 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.b 2024-04-13T07:51:41.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:40 smithi203 systemd[1]: Stopping Ceph mon.b for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:51:41.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:40 smithi203 bash[68060]: debug 2024-04-13T07:51:40.956+0000 7faeb8728700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-stderr=true (PID: 1) UID: 0 2024-04-13T07:51:41.019 INFO:journalctl@ceph.mon.b.smithi203.stdout:Apr 13 07:51:40 smithi203 bash[68060]: debug 2024-04-13T07:51:40.956+0000 7faeb8728700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2024-04-13T07:51:41.980 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mon.b.service' 2024-04-13T07:51:42.013 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:51:42.014 INFO:tasks.cephadm.mon.b:Stopped mon.b 2024-04-13T07:51:42.014 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2024-04-13T07:51:42.014 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y 2024-04-13T07:51:42.301 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:42 smithi089 systemd[1]: Stopping Ceph mgr.y for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:51:42.301 INFO:journalctl@ceph.mgr.y.smithi089.stdout:Apr 13 07:51:42 smithi089 bash[74725]: debug 2024-04-13T07:51:42.163+0000 7fab18aff700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:51:42.519 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:51:42 smithi203 bash[60252]: ts=2024-04-13T07:51:42.315Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=pools name=CephPoolGrowthWarning index=0 msg="Evaluating rule failed" rule="alert: CephPoolGrowthWarning\nexpr: (predict_linear(ceph_pool_percent_used[2d], 3600 * 24 * 5) * on (pool_id, instance)\n group_right () ceph_pool_metadata) >= 95\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.9.2\n severity: warning\n type: ceph_default\nannotations:\n description: Pool '{{ $labels.name }}' will be full in less than 5 days assuming\n the average fill-up rate of the past 48 hours.\n summary: Pool growth rate may soon exceed capacity\n" err="found duplicate series for the match group {instance=\"smithi089:9283\", pool_id=\"1\"} on the left hand-side of the operation: [{instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}, {cluster=\"ec50a262-f965-11ee-bc8d-c7b262605968\", instance=\"smithi089:9283\", job=\"ceph\", pool_id=\"1\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2024-04-13T07:51:43.032 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.y.service' 2024-04-13T07:51:43.055 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:51:43.055 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2024-04-13T07:51:43.055 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2024-04-13T07:51:43.055 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.x 2024-04-13T07:51:43.499 INFO:journalctl@ceph.mgr.x.smithi203.stdout:Apr 13 07:51:43 smithi203 systemd[1]: Stopping Ceph mgr.x for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:51:44.090 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@mgr.x.service' 2024-04-13T07:51:44.114 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:51:44.115 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2024-04-13T07:51:44.115 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2024-04-13T07:51:44.115 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.0 2024-04-13T07:51:44.551 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:51:44 smithi089 systemd[1]: Stopping Ceph osd.0 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:51:44.552 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:51:44 smithi089 bash[90890]: debug 2024-04-13T07:51:44.259+0000 7fb7cec1d700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:51:44.552 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:51:44 smithi089 bash[90890]: debug 2024-04-13T07:51:44.259+0000 7fb7cec1d700 -1 osd.0 117 *** Got signal Terminated *** 2024-04-13T07:51:44.552 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:51:44 smithi089 bash[90890]: debug 2024-04-13T07:51:44.259+0000 7fb7cec1d700 -1 osd.0 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:51:50.443 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:51:50 smithi089 bash[121624]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-0 2024-04-13T07:51:50.444 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:51:50 smithi089 bash[58201]: ts=2024-04-13T07:51:50.174Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:51:50.444 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:51:50 smithi089 bash[58201]: ts=2024-04-13T07:51:50.175Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:51:50.801 INFO:journalctl@ceph.osd.0.smithi089.stdout:Apr 13 07:51:50 smithi089 bash[121688]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-0 2024-04-13T07:51:52.291 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.0.service' 2024-04-13T07:51:52.321 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:51:52.321 INFO:tasks.cephadm.osd.0:Stopped osd.0 2024-04-13T07:51:52.322 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2024-04-13T07:51:52.322 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.1 2024-04-13T07:51:52.551 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:51:52 smithi089 systemd[1]: Stopping Ceph osd.1 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:51:52.551 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:51:52 smithi089 bash[96407]: debug 2024-04-13T07:51:52.499+0000 7f7c2d0d7700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:51:52.552 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:51:52 smithi089 bash[96407]: debug 2024-04-13T07:51:52.499+0000 7f7c2d0d7700 -1 osd.1 117 *** Got signal Terminated *** 2024-04-13T07:51:52.552 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:51:52 smithi089 bash[96407]: debug 2024-04-13T07:51:52.499+0000 7f7c2d0d7700 -1 osd.1 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:51:58.674 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:51:58 smithi089 bash[121825]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-1 2024-04-13T07:51:58.674 INFO:journalctl@ceph.osd.1.smithi089.stdout:Apr 13 07:51:58 smithi089 bash[121889]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-1 2024-04-13T07:52:00.365 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.1.service' 2024-04-13T07:52:00.393 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:52:00.393 INFO:tasks.cephadm.osd.1:Stopped osd.1 2024-04-13T07:52:00.394 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2024-04-13T07:52:00.394 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.2 2024-04-13T07:52:00.447 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:00 smithi089 bash[58201]: ts=2024-04-13T07:52:00.173Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:00.448 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:00 smithi089 bash[58201]: ts=2024-04-13T07:52:00.174Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:00.448 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:00 smithi089 bash[58201]: ts=2024-04-13T07:52:00.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:52:00.801 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:52:00 smithi089 systemd[1]: Stopping Ceph osd.2 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:00.802 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:52:00 smithi089 bash[101030]: debug 2024-04-13T07:52:00.587+0000 7fea69499700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:52:00.802 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:52:00 smithi089 bash[101030]: debug 2024-04-13T07:52:00.587+0000 7fea69499700 -1 osd.2 117 *** Got signal Terminated *** 2024-04-13T07:52:00.802 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:52:00 smithi089 bash[101030]: debug 2024-04-13T07:52:00.587+0000 7fea69499700 -1 osd.2 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:52:07.051 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:52:06 smithi089 bash[122024]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-2 2024-04-13T07:52:07.051 INFO:journalctl@ceph.osd.2.smithi089.stdout:Apr 13 07:52:06 smithi089 bash[122086]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-2 2024-04-13T07:52:08.638 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.2.service' 2024-04-13T07:52:08.709 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:52:08.709 INFO:tasks.cephadm.osd.2:Stopped osd.2 2024-04-13T07:52:08.709 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2024-04-13T07:52:08.710 DEBUG:teuthology.orchestra.run.smithi089:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.3 2024-04-13T07:52:08.893 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:52:08 smithi089 systemd[1]: Stopping Ceph osd.3 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:09.301 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:52:08 smithi089 bash[105315]: debug 2024-04-13T07:52:08.894+0000 7f62ae73e700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:52:09.301 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:52:08 smithi089 bash[105315]: debug 2024-04-13T07:52:08.894+0000 7f62ae73e700 -1 osd.3 117 *** Got signal Terminated *** 2024-04-13T07:52:09.301 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:52:08 smithi089 bash[105315]: debug 2024-04-13T07:52:08.894+0000 7f62ae73e700 -1 osd.3 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:52:10.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:10 smithi089 bash[58201]: ts=2024-04-13T07:52:10.175Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:10.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:10 smithi089 bash[58201]: ts=2024-04-13T07:52:10.175Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:10.552 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:10 smithi089 bash[58201]: ts=2024-04-13T07:52:10.177Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:52:11.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:11 smithi203 bash[73720]: debug 2024-04-13T07:52:11.044+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:12.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:12 smithi203 bash[73720]: debug 2024-04-13T07:52:12.043+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:12.801 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:52:12 smithi089 bash[105315]: debug 2024-04-13T07:52:12.398+0000 7f62a6d2a700 -1 osd.3 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:46.459968+0000 front 2024-04-13T07:51:46.460453+0000 (oldest deadline 2024-04-13T07:52:12.359905+0000) 2024-04-13T07:52:13.269 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:13 smithi203 bash[73720]: debug 2024-04-13T07:52:13.043+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:13.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:13 smithi203 bash[43978]: debug 2024-04-13T07:52:13.015+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:13.270 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:12 smithi203 bash[38192]: debug 2024-04-13T07:52:12.891+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:13.801 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:52:13 smithi089 bash[105315]: debug 2024-04-13T07:52:13.354+0000 7f62a6d2a700 -1 osd.3 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:46.459968+0000 front 2024-04-13T07:51:46.460453+0000 (oldest deadline 2024-04-13T07:52:12.359905+0000) 2024-04-13T07:52:14.269 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:14 smithi203 bash[73720]: debug 2024-04-13T07:52:14.087+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:14.270 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:13 smithi203 bash[38192]: debug 2024-04-13T07:52:13.867+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:14.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:14 smithi203 bash[43978]: debug 2024-04-13T07:52:14.019+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:15.051 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:52:14 smithi089 bash[122217]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-3 2024-04-13T07:52:15.051 INFO:journalctl@ceph.osd.3.smithi089.stdout:Apr 13 07:52:14 smithi089 bash[122281]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-3 2024-04-13T07:52:15.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:15 smithi203 bash[43978]: debug 2024-04-13T07:52:15.055+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:15.269 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:15 smithi203 bash[73720]: debug 2024-04-13T07:52:15.131+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:15.270 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:14 smithi203 bash[38192]: debug 2024-04-13T07:52:14.891+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:16.177 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:15 smithi203 bash[38192]: debug 2024-04-13T07:52:15.863+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:16.178 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:16 smithi203 bash[43978]: debug 2024-04-13T07:52:16.007+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:16.178 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:15 smithi203 bash[32709]: debug 2024-04-13T07:52:15.911+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:16.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:16 smithi203 bash[73720]: debug 2024-04-13T07:52:16.175+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:16.712 DEBUG:teuthology.orchestra.run.smithi089:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.3.service' 2024-04-13T07:52:16.739 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:52:16.740 INFO:tasks.cephadm.osd.3:Stopped osd.3 2024-04-13T07:52:16.740 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2024-04-13T07:52:16.740 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.4 2024-04-13T07:52:17.003 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:16 smithi203 bash[43978]: debug 2024-04-13T07:52:16.979+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:17.003 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:17 smithi203 bash[60252]: ts=2024-04-13T07:52:17.001Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-04-13T07:52:17.004 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:16 smithi203 systemd[1]: Stopping Ceph osd.4 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:17.004 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:16 smithi203 bash[73720]: debug 2024-04-13T07:52:16.971+0000 7f2ab7fba700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:52:17.004 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:16 smithi203 bash[73720]: debug 2024-04-13T07:52:16.971+0000 7f2ab7fba700 -1 osd.4 117 *** Got signal Terminated *** 2024-04-13T07:52:17.004 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:16 smithi203 bash[73720]: debug 2024-04-13T07:52:16.971+0000 7f2ab7fba700 -1 osd.4 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:52:17.005 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:16 smithi203 bash[32709]: debug 2024-04-13T07:52:16.903+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:17.005 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:16 smithi203 bash[38192]: debug 2024-04-13T07:52:16.855+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:17.019 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:17 smithi203 bash[60252]: ts=2024-04-13T07:52:17.005Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=nvmeof msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=nvmeof\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-04-13T07:52:17.019 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:17 smithi203 bash[60252]: ts=2024-04-13T07:52:17.005Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-04-13T07:52:17.019 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:17 smithi203 bash[60252]: ts=2024-04-13T07:52:17.007Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-04-13T07:52:17.019 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:17 smithi203 bash[60252]: ts=2024-04-13T07:52:17.008Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.89:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.89:8765: connect: connection refused" 2024-04-13T07:52:17.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:17 smithi203 bash[73720]: debug 2024-04-13T07:52:17.159+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:18.123 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:17 smithi203 bash[32709]: debug 2024-04-13T07:52:17.907+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:18.124 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:18 smithi203 bash[43978]: debug 2024-04-13T07:52:18.027+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:18.124 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:17 smithi203 bash[38192]: debug 2024-04-13T07:52:17.827+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:18.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:18 smithi203 bash[73720]: debug 2024-04-13T07:52:18.123+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:19.069 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:18 smithi203 bash[32709]: debug 2024-04-13T07:52:18.927+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:19.070 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:18 smithi203 bash[38192]: debug 2024-04-13T07:52:18.807+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:19.070 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:18 smithi203 bash[38192]: debug 2024-04-13T07:52:18.807+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:19.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:19 smithi203 bash[73720]: debug 2024-04-13T07:52:19.091+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:19.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:19 smithi203 bash[43978]: debug 2024-04-13T07:52:19.067+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:20.127 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:19 smithi203 bash[32709]: debug 2024-04-13T07:52:19.907+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:20.127 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:20 smithi203 bash[43978]: debug 2024-04-13T07:52:20.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:20.128 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:19 smithi203 bash[38192]: debug 2024-04-13T07:52:19.783+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:20.128 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:19 smithi203 bash[38192]: debug 2024-04-13T07:52:19.783+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:20.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:20 smithi203 bash[73720]: debug 2024-04-13T07:52:20.123+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:20.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:20 smithi089 bash[58201]: ts=2024-04-13T07:52:20.175Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:20.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:20 smithi089 bash[58201]: ts=2024-04-13T07:52:20.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:20.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:20 smithi089 bash[58201]: ts=2024-04-13T07:52:20.177Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:52:21.081 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:20 smithi203 bash[32709]: debug 2024-04-13T07:52:20.951+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:21.081 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:20 smithi203 bash[32709]: debug 2024-04-13T07:52:20.951+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:21.081 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:20 smithi203 bash[38192]: debug 2024-04-13T07:52:20.807+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:21.081 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:20 smithi203 bash[38192]: debug 2024-04-13T07:52:20.807+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:21.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:21 smithi203 bash[43978]: debug 2024-04-13T07:52:21.079+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:21.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:21 smithi203 bash[73720]: debug 2024-04-13T07:52:21.147+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:45.619495+0000 front 2024-04-13T07:51:45.620010+0000 (oldest deadline 2024-04-13T07:52:10.919244+0000) 2024-04-13T07:52:21.520 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:21 smithi203 bash[73720]: debug 2024-04-13T07:52:21.147+0000 7f2ab05a6700 -1 osd.4 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:55.021097+0000 front 2024-04-13T07:51:55.021185+0000 (oldest deadline 2024-04-13T07:52:20.920645+0000) 2024-04-13T07:52:22.019 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:21 smithi203 bash[32709]: debug 2024-04-13T07:52:21.963+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:22.019 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:21 smithi203 bash[32709]: debug 2024-04-13T07:52:21.963+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:22.020 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:21 smithi203 bash[38192]: debug 2024-04-13T07:52:21.763+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:22.020 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:21 smithi203 bash[38192]: debug 2024-04-13T07:52:21.763+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:22.373 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:22 smithi203 bash[43978]: debug 2024-04-13T07:52:22.103+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:23.019 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:22 smithi203 bash[79820]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-4 2024-04-13T07:52:23.019 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:22 smithi203 bash[32709]: debug 2024-04-13T07:52:22.971+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:23.019 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:22 smithi203 bash[32709]: debug 2024-04-13T07:52:22.971+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:23.020 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:22 smithi203 bash[38192]: debug 2024-04-13T07:52:22.759+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:23.020 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:22 smithi203 bash[38192]: debug 2024-04-13T07:52:22.759+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:23.519 INFO:journalctl@ceph.osd.4.smithi203.stdout:Apr 13 07:52:23 smithi203 bash[79883]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-4 2024-04-13T07:52:23.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:23 smithi203 bash[43978]: debug 2024-04-13T07:52:23.099+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:23.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:23 smithi203 bash[43978]: debug 2024-04-13T07:52:23.099+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:24.088 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:24 smithi203 bash[32709]: debug 2024-04-13T07:52:23.999+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:24.089 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:24 smithi203 bash[32709]: debug 2024-04-13T07:52:23.999+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:24.089 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:23 smithi203 bash[38192]: debug 2024-04-13T07:52:23.787+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:24.089 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:23 smithi203 bash[38192]: debug 2024-04-13T07:52:23.787+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:24.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:24 smithi203 bash[43978]: debug 2024-04-13T07:52:24.083+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:24.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:24 smithi203 bash[43978]: debug 2024-04-13T07:52:24.083+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:25.011 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:24 smithi203 bash[38192]: debug 2024-04-13T07:52:24.743+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:25.012 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:24 smithi203 bash[38192]: debug 2024-04-13T07:52:24.743+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:25.269 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[32709]: debug 2024-04-13T07:52:25.007+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:25.269 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[32709]: debug 2024-04-13T07:52:25.007+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:25.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[43978]: debug 2024-04-13T07:52:25.075+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:25.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[43978]: debug 2024-04-13T07:52:25.075+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:25.286 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.4.service' 2024-04-13T07:52:25.315 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:52:25.315 INFO:tasks.cephadm.osd.4:Stopped osd.4 2024-04-13T07:52:25.315 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2024-04-13T07:52:25.316 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.5 2024-04-13T07:52:25.769 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:25 smithi203 systemd[1]: Stopping Ceph osd.5 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:25.769 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[32709]: debug 2024-04-13T07:52:25.495+0000 7f0618504700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:52:25.769 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[32709]: debug 2024-04-13T07:52:25.495+0000 7f0618504700 -1 osd.5 117 *** Got signal Terminated *** 2024-04-13T07:52:25.770 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[32709]: debug 2024-04-13T07:52:25.495+0000 7f0618504700 -1 osd.5 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:52:26.081 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:26 smithi203 bash[32709]: debug 2024-04-13T07:52:26.011+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:26.081 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:26 smithi203 bash[32709]: debug 2024-04-13T07:52:26.011+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:26.081 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[38192]: debug 2024-04-13T07:52:25.779+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:26.082 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:25 smithi203 bash[38192]: debug 2024-04-13T07:52:25.779+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:26.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:26 smithi203 bash[43978]: debug 2024-04-13T07:52:26.083+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:26.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:26 smithi203 bash[43978]: debug 2024-04-13T07:52:26.083+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:27.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:26 smithi203 bash[38192]: debug 2024-04-13T07:52:26.735+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:27.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:26 smithi203 bash[38192]: debug 2024-04-13T07:52:26.735+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:27.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:27 smithi203 bash[43978]: debug 2024-04-13T07:52:27.111+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:27.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:27 smithi203 bash[43978]: debug 2024-04-13T07:52:27.111+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:27.520 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:27 smithi203 bash[32709]: debug 2024-04-13T07:52:27.055+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:27.520 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:27 smithi203 bash[32709]: debug 2024-04-13T07:52:27.055+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:27.520 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:27 smithi203 bash[32709]: debug 2024-04-13T07:52:27.055+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.297635+0000 front 2024-04-13T07:52:05.297350+0000 (oldest deadline 2024-04-13T07:52:26.397237+0000) 2024-04-13T07:52:28.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:27 smithi203 bash[38192]: debug 2024-04-13T07:52:27.711+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:28.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:27 smithi203 bash[38192]: debug 2024-04-13T07:52:27.711+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:28.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:28 smithi203 bash[32709]: debug 2024-04-13T07:52:28.075+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:28.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:28 smithi203 bash[32709]: debug 2024-04-13T07:52:28.075+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:28.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:28 smithi203 bash[32709]: debug 2024-04-13T07:52:28.075+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.297635+0000 front 2024-04-13T07:52:05.297350+0000 (oldest deadline 2024-04-13T07:52:26.397237+0000) 2024-04-13T07:52:28.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:28 smithi203 bash[43978]: debug 2024-04-13T07:52:28.075+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:28.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:28 smithi203 bash[43978]: debug 2024-04-13T07:52:28.075+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:29.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:28 smithi203 bash[38192]: debug 2024-04-13T07:52:28.671+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:29.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:28 smithi203 bash[38192]: debug 2024-04-13T07:52:28.671+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:29.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[32709]: debug 2024-04-13T07:52:29.079+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:29.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[32709]: debug 2024-04-13T07:52:29.079+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:29.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[32709]: debug 2024-04-13T07:52:29.079+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.297635+0000 front 2024-04-13T07:52:05.297350+0000 (oldest deadline 2024-04-13T07:52:26.397237+0000) 2024-04-13T07:52:29.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[43978]: debug 2024-04-13T07:52:29.083+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:29.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[43978]: debug 2024-04-13T07:52:29.083+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:29.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[43978]: debug 2024-04-13T07:52:29.083+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:30.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[38192]: debug 2024-04-13T07:52:29.727+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:30.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[38192]: debug 2024-04-13T07:52:29.727+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:30.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:29 smithi203 bash[38192]: debug 2024-04-13T07:52:29.727+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:30.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[32709]: debug 2024-04-13T07:52:30.103+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.193148+0000 front 2024-04-13T07:51:49.193500+0000 (oldest deadline 2024-04-13T07:52:15.092883+0000) 2024-04-13T07:52:30.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[32709]: debug 2024-04-13T07:52:30.103+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.194534+0000 front 2024-04-13T07:51:56.194287+0000 (oldest deadline 2024-04-13T07:52:20.894320+0000) 2024-04-13T07:52:30.519 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[32709]: debug 2024-04-13T07:52:30.103+0000 7f0614afa700 -1 osd.5 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.297635+0000 front 2024-04-13T07:52:05.297350+0000 (oldest deadline 2024-04-13T07:52:26.397237+0000) 2024-04-13T07:52:30.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[43978]: debug 2024-04-13T07:52:30.055+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:30.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[43978]: debug 2024-04-13T07:52:30.055+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:30.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[43978]: debug 2024-04-13T07:52:30.055+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:30.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:30 smithi089 bash[58201]: ts=2024-04-13T07:52:30.176Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 8 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:30.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:30 smithi089 bash[58201]: ts=2024-04-13T07:52:30.177Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:30.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:30 smithi089 bash[58201]: ts=2024-04-13T07:52:30.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:52:30.954 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[38192]: debug 2024-04-13T07:52:30.687+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:30.954 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[38192]: debug 2024-04-13T07:52:30.687+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:30.954 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:30 smithi203 bash[38192]: debug 2024-04-13T07:52:30.687+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:31.309 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:31 smithi203 bash[43978]: debug 2024-04-13T07:52:31.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:31.309 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:31 smithi203 bash[43978]: debug 2024-04-13T07:52:31.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:31.309 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:31 smithi203 bash[43978]: debug 2024-04-13T07:52:31.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:32.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:31 smithi203 bash[38192]: debug 2024-04-13T07:52:31.671+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:32.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:31 smithi203 bash[38192]: debug 2024-04-13T07:52:31.671+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:32.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:31 smithi203 bash[38192]: debug 2024-04-13T07:52:31.671+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:32.020 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:31 smithi203 bash[80012]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-5 2024-04-13T07:52:32.020 INFO:journalctl@ceph.osd.5.smithi203.stdout:Apr 13 07:52:31 smithi203 bash[80078]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-5 2024-04-13T07:52:32.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:32 smithi203 bash[43978]: debug 2024-04-13T07:52:32.035+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:32.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:32 smithi203 bash[43978]: debug 2024-04-13T07:52:32.035+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:32.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:32 smithi203 bash[43978]: debug 2024-04-13T07:52:32.035+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:33.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:32 smithi203 bash[38192]: debug 2024-04-13T07:52:32.651+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:33.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:32 smithi203 bash[38192]: debug 2024-04-13T07:52:32.651+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:33.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:32 smithi203 bash[38192]: debug 2024-04-13T07:52:32.651+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:33.431 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:33 smithi203 bash[43978]: debug 2024-04-13T07:52:33.051+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:33.432 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:33 smithi203 bash[43978]: debug 2024-04-13T07:52:33.051+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:33.432 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:33 smithi203 bash[43978]: debug 2024-04-13T07:52:33.051+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:33.769 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:33 smithi203 bash[38192]: debug 2024-04-13T07:52:33.659+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:33.769 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:33 smithi203 bash[38192]: debug 2024-04-13T07:52:33.659+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:33.769 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:33 smithi203 bash[38192]: debug 2024-04-13T07:52:33.659+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:33.904 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.5.service' 2024-04-13T07:52:33.931 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:52:33.931 INFO:tasks.cephadm.osd.5:Stopped osd.5 2024-04-13T07:52:33.931 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2024-04-13T07:52:33.932 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.6 2024-04-13T07:52:34.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[43978]: debug 2024-04-13T07:52:34.043+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:34.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[43978]: debug 2024-04-13T07:52:34.043+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:34.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[43978]: debug 2024-04-13T07:52:34.043+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:34.270 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:33 smithi203 systemd[1]: Stopping Ceph osd.6 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:34.270 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[38192]: debug 2024-04-13T07:52:34.107+0000 7fa238898700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:52:34.270 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[38192]: debug 2024-04-13T07:52:34.107+0000 7fa238898700 -1 osd.6 117 *** Got signal Terminated *** 2024-04-13T07:52:34.270 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[38192]: debug 2024-04-13T07:52:34.107+0000 7fa238898700 -1 osd.6 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:52:35.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[38192]: debug 2024-04-13T07:52:34.611+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:35.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[38192]: debug 2024-04-13T07:52:34.611+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:35.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:34 smithi203 bash[38192]: debug 2024-04-13T07:52:34.611+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:35.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:35 smithi203 bash[43978]: debug 2024-04-13T07:52:35.055+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:35.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:35 smithi203 bash[43978]: debug 2024-04-13T07:52:35.055+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:35.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:35 smithi203 bash[43978]: debug 2024-04-13T07:52:35.055+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:36.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:35 smithi203 bash[38192]: debug 2024-04-13T07:52:35.655+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:36.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:35 smithi203 bash[38192]: debug 2024-04-13T07:52:35.655+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:36.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:35 smithi203 bash[38192]: debug 2024-04-13T07:52:35.655+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:36.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:35 smithi203 bash[38192]: debug 2024-04-13T07:52:35.655+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:10.678692+0000 front 2024-04-13T07:52:10.678746+0000 (oldest deadline 2024-04-13T07:52:34.778573+0000) 2024-04-13T07:52:36.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:36 smithi203 bash[43978]: debug 2024-04-13T07:52:36.059+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:36.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:36 smithi203 bash[43978]: debug 2024-04-13T07:52:36.059+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:36.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:36 smithi203 bash[43978]: debug 2024-04-13T07:52:36.059+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:37.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:36 smithi203 bash[38192]: debug 2024-04-13T07:52:36.607+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:37.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:36 smithi203 bash[38192]: debug 2024-04-13T07:52:36.607+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:37.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:36 smithi203 bash[38192]: debug 2024-04-13T07:52:36.607+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:37.020 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:36 smithi203 bash[38192]: debug 2024-04-13T07:52:36.607+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:10.678692+0000 front 2024-04-13T07:52:10.678746+0000 (oldest deadline 2024-04-13T07:52:34.778573+0000) 2024-04-13T07:52:37.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:37 smithi203 bash[43978]: debug 2024-04-13T07:52:37.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:37.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:37 smithi203 bash[43978]: debug 2024-04-13T07:52:37.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:37.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:37 smithi203 bash[43978]: debug 2024-04-13T07:52:37.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:38.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:37 smithi203 bash[38192]: debug 2024-04-13T07:52:37.587+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:38.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:37 smithi203 bash[38192]: debug 2024-04-13T07:52:37.587+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:38.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:37 smithi203 bash[38192]: debug 2024-04-13T07:52:37.587+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:38.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:37 smithi203 bash[38192]: debug 2024-04-13T07:52:37.587+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:10.678692+0000 front 2024-04-13T07:52:10.678746+0000 (oldest deadline 2024-04-13T07:52:34.778573+0000) 2024-04-13T07:52:38.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:38 smithi203 bash[43978]: debug 2024-04-13T07:52:38.023+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:38.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:38 smithi203 bash[43978]: debug 2024-04-13T07:52:38.023+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:38.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:38 smithi203 bash[43978]: debug 2024-04-13T07:52:38.023+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:38.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:38 smithi203 bash[43978]: debug 2024-04-13T07:52:38.023+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:39.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:38 smithi203 bash[38192]: debug 2024-04-13T07:52:38.567+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:49.174364+0000 front 2024-04-13T07:51:49.174441+0000 (oldest deadline 2024-04-13T07:52:12.074096+0000) 2024-04-13T07:52:39.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:38 smithi203 bash[38192]: debug 2024-04-13T07:52:38.567+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:53.175731+0000 front 2024-04-13T07:51:53.175618+0000 (oldest deadline 2024-04-13T07:52:18.475470+0000) 2024-04-13T07:52:39.019 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:38 smithi203 bash[38192]: debug 2024-04-13T07:52:38.567+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:05.477474+0000 front 2024-04-13T07:52:05.477639+0000 (oldest deadline 2024-04-13T07:52:29.577265+0000) 2024-04-13T07:52:39.020 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:38 smithi203 bash[38192]: debug 2024-04-13T07:52:38.567+0000 7fa234e8e700 -1 osd.6 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:10.678692+0000 front 2024-04-13T07:52:10.678746+0000 (oldest deadline 2024-04-13T07:52:34.778573+0000) 2024-04-13T07:52:39.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:39 smithi203 bash[43978]: debug 2024-04-13T07:52:39.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:39.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:39 smithi203 bash[43978]: debug 2024-04-13T07:52:39.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:39.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:39 smithi203 bash[43978]: debug 2024-04-13T07:52:39.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:39.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:39 smithi203 bash[43978]: debug 2024-04-13T07:52:39.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:40.241 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:39 smithi203 bash[43978]: debug 2024-04-13T07:52:39.983+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:40.242 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:39 smithi203 bash[43978]: debug 2024-04-13T07:52:39.983+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:40.242 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:39 smithi203 bash[43978]: debug 2024-04-13T07:52:39.983+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:40.242 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:39 smithi203 bash[43978]: debug 2024-04-13T07:52:39.983+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:40.496 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:40 smithi203 bash[80210]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-6 2024-04-13T07:52:40.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:40 smithi089 bash[58201]: ts=2024-04-13T07:52:40.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused; ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:40.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:40 smithi089 bash[58201]: ts=2024-04-13T07:52:40.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:40.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:40 smithi089 bash[58201]: ts=2024-04-13T07:52:40.179Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:52:40.769 INFO:journalctl@ceph.osd.6.smithi203.stdout:Apr 13 07:52:40 smithi203 bash[80274]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-6 2024-04-13T07:52:41.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:40 smithi203 bash[43978]: debug 2024-04-13T07:52:40.955+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:41.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:40 smithi203 bash[43978]: debug 2024-04-13T07:52:40.955+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:41.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:40 smithi203 bash[43978]: debug 2024-04-13T07:52:40.955+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:41.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:40 smithi203 bash[43978]: debug 2024-04-13T07:52:40.955+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:42.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:41 smithi203 bash[43978]: debug 2024-04-13T07:52:41.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:42.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:41 smithi203 bash[43978]: debug 2024-04-13T07:52:41.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:42.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:41 smithi203 bash[43978]: debug 2024-04-13T07:52:41.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:42.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:41 smithi203 bash[43978]: debug 2024-04-13T07:52:41.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:42.512 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.6.service' 2024-04-13T07:52:42.539 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:52:42.539 INFO:tasks.cephadm.osd.6:Stopped osd.6 2024-04-13T07:52:42.539 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2024-04-13T07:52:42.539 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.7 2024-04-13T07:52:42.769 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:42 smithi203 systemd[1]: Stopping Ceph osd.7 for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:42.769 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:42 smithi203 bash[43978]: debug 2024-04-13T07:52:42.699+0000 7fe54920f700 -1 received signal: Terminated from /sbin/docker-init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-stderr=true --default-log-stderr-prefix=debug (PID: 1) UID: 0 2024-04-13T07:52:42.769 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:42 smithi203 bash[43978]: debug 2024-04-13T07:52:42.699+0000 7fe54920f700 -1 osd.7 117 *** Got signal Terminated *** 2024-04-13T07:52:42.770 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:42 smithi203 bash[43978]: debug 2024-04-13T07:52:42.699+0000 7fe54920f700 -1 osd.7 117 *** Immediate shutdown (osd_fast_shutdown=true) *** 2024-04-13T07:52:43.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:43.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:43.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:43.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:43.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.203:6804 osd.4 since back 2024-04-13T07:52:17.763878+0000 front 2024-04-13T07:52:17.763761+0000 (oldest deadline 2024-04-13T07:52:42.463374+0000) 2024-04-13T07:52:44.269 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:44.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:44.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:44.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:44.270 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:43 smithi203 bash[43978]: debug 2024-04-13T07:52:43.987+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.203:6804 osd.4 since back 2024-04-13T07:52:17.763878+0000 front 2024-04-13T07:52:17.763761+0000 (oldest deadline 2024-04-13T07:52:42.463374+0000) 2024-04-13T07:52:45.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:45 smithi203 bash[43978]: debug 2024-04-13T07:52:45.039+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:45.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:45 smithi203 bash[43978]: debug 2024-04-13T07:52:45.039+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:45.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:45 smithi203 bash[43978]: debug 2024-04-13T07:52:45.039+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:45.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:45 smithi203 bash[43978]: debug 2024-04-13T07:52:45.039+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:45.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:45 smithi203 bash[43978]: debug 2024-04-13T07:52:45.039+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.203:6804 osd.4 since back 2024-04-13T07:52:17.763878+0000 front 2024-04-13T07:52:17.763761+0000 (oldest deadline 2024-04-13T07:52:42.463374+0000) 2024-04-13T07:52:46.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:46 smithi203 bash[43978]: debug 2024-04-13T07:52:46.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:46.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:46 smithi203 bash[43978]: debug 2024-04-13T07:52:46.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:46.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:46 smithi203 bash[43978]: debug 2024-04-13T07:52:46.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:46.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:46 smithi203 bash[43978]: debug 2024-04-13T07:52:46.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:46.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:46 smithi203 bash[43978]: debug 2024-04-13T07:52:46.031+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.203:6804 osd.4 since back 2024-04-13T07:52:17.763878+0000 front 2024-04-13T07:52:17.763761+0000 (oldest deadline 2024-04-13T07:52:42.463374+0000) 2024-04-13T07:52:47.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:47 smithi203 bash[43978]: debug 2024-04-13T07:52:47.071+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6806 osd.0 since back 2024-04-13T07:51:47.859263+0000 front 2024-04-13T07:51:47.859432+0000 (oldest deadline 2024-04-13T07:52:12.559306+0000) 2024-04-13T07:52:47.519 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:47 smithi203 bash[43978]: debug 2024-04-13T07:52:47.071+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6814 osd.1 since back 2024-04-13T07:51:56.561600+0000 front 2024-04-13T07:51:56.561202+0000 (oldest deadline 2024-04-13T07:52:22.461195+0000) 2024-04-13T07:52:47.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:47 smithi203 bash[43978]: debug 2024-04-13T07:52:47.071+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6822 osd.2 since back 2024-04-13T07:52:02.462104+0000 front 2024-04-13T07:52:02.462219+0000 (oldest deadline 2024-04-13T07:52:28.361743+0000) 2024-04-13T07:52:47.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:47 smithi203 bash[43978]: debug 2024-04-13T07:52:47.071+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.89:6830 osd.3 since back 2024-04-13T07:52:13.663331+0000 front 2024-04-13T07:52:13.663213+0000 (oldest deadline 2024-04-13T07:52:37.762893+0000) 2024-04-13T07:52:47.520 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:47 smithi203 bash[43978]: debug 2024-04-13T07:52:47.071+0000 7fe5417fd700 -1 osd.7 117 heartbeat_check: no reply from 172.21.15.203:6804 osd.4 since back 2024-04-13T07:52:17.763878+0000 front 2024-04-13T07:52:17.763761+0000 (oldest deadline 2024-04-13T07:52:42.463374+0000) 2024-04-13T07:52:48.917 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:48 smithi203 bash[80408]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-7 2024-04-13T07:52:48.917 INFO:journalctl@ceph.osd.7.smithi203.stdout:Apr 13 07:52:48 smithi203 bash[80473]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-osd-7 2024-04-13T07:52:50.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:50 smithi089 bash[58201]: ts=2024-04-13T07:52:50.178Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[0]: notify retry canceled after 7 attempts: Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused; ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:52:50.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:50 smithi089 bash[58201]: ts=2024-04-13T07:52:50.179Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi089.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.89:8443: connect: connection refused" 2024-04-13T07:52:50.551 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:50 smithi089 bash[58201]: ts=2024-04-13T07:52:50.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi203.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.203:8443: connect: connection refused" 2024-04-13T07:52:50.903 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@osd.7.service' 2024-04-13T07:52:50.930 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:52:50.930 INFO:tasks.cephadm.osd.7:Stopped osd.7 2024-04-13T07:52:50.930 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2024-04-13T07:52:50.931 DEBUG:teuthology.orchestra.run.smithi203:> sudo systemctl stop ceph-ec50a262-f965-11ee-bc8d-c7b262605968@prometheus.a 2024-04-13T07:52:51.166 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:50 smithi203 systemd[1]: Stopping Ceph prometheus.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:51.166 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.121Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2024-04-13T07:52:51.166 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.121Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2024-04-13T07:52:51.166 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.121Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2024-04-13T07:52:51.166 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.121Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2024-04-13T07:52:51.166 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.121Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2024-04-13T07:52:51.167 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.121Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2024-04-13T07:52:51.167 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.121Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2024-04-13T07:52:51.167 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.121Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2024-04-13T07:52:51.167 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.122Z caller=main.go:907 level=info msg="Scrape manager stopped" 2024-04-13T07:52:51.167 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.122Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2024-04-13T07:52:51.167 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.122Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2024-04-13T07:52:51.167 INFO:journalctl@ceph.prometheus.a.smithi203.stdout:Apr 13 07:52:51 smithi203 bash[60252]: ts=2024-04-13T07:52:51.122Z caller=main.go:1147 level=info msg="See you next time!" 2024-04-13T07:52:51.901 DEBUG:teuthology.orchestra.run.smithi203:> sudo pkill -f 'journalctl -f -n 0 -u ceph-ec50a262-f965-11ee-bc8d-c7b262605968@prometheus.a.service' 2024-04-13T07:52:51.922 DEBUG:teuthology.orchestra.run:got remote process result: None 2024-04-13T07:52:51.923 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2024-04-13T07:52:51.923 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ec50a262-f965-11ee-bc8d-c7b262605968 --force --keep-logs 2024-04-13T07:52:52.048 INFO:teuthology.orchestra.run.smithi089.stdout:Deleting cluster with fsid: ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:52:56.879 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:56 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:56.879 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:56 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:57.232 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:57.232 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:57.518 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:57.519 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:57.519 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:57.519 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:57.787 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:57.788 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:58.051 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:58.051 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:57 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:58.439 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:58 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:58.439 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:58 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:58.440 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:58 smithi089 systemd[1]: Stopping Ceph node-exporter.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:59.255 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:58 smithi089 bash[122640]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-node-exporter-a 2024-04-13T07:52:59.255 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:59 smithi089 bash[122702]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-node-exporter-a 2024-04-13T07:52:59.256 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:59 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2024-04-13T07:52:59.256 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:59 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@node-exporter.a.service: Failed with result 'exit-code'. 2024-04-13T07:52:59.256 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:59 smithi089 systemd[1]: Stopped Ceph node-exporter.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:52:59.528 INFO:journalctl@ceph.node-exporter.a.smithi089.stdout:Apr 13 07:52:59 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:59.528 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:59 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:52:59.528 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:59 smithi089 systemd[1]: Stopping Ceph alertmanager.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:52:59.529 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:52:59 smithi089 bash[58201]: ts=2024-04-13T07:52:59.485Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2024-04-13T07:53:00.301 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:53:00 smithi089 bash[122767]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-alertmanager-a 2024-04-13T07:53:00.301 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:53:00 smithi089 bash[122826]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-alertmanager-a 2024-04-13T07:53:00.301 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:53:00 smithi089 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@alertmanager.a.service: Deactivated successfully. 2024-04-13T07:53:00.301 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:53:00 smithi089 systemd[1]: Stopped Ceph alertmanager.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:53:00.801 INFO:journalctl@ceph.alertmanager.a.smithi089.stdout:Apr 13 07:53:00 smithi089 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:04.206 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ec50a262-f965-11ee-bc8d-c7b262605968 --force --keep-logs 2024-04-13T07:53:04.331 INFO:teuthology.orchestra.run.smithi203.stdout:Deleting cluster with fsid: ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:53:09.171 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:09.171 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:09.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:09.519 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:09.840 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:09.840 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:09.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:09.841 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:10.110 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:09 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:10.111 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:10 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:10.443 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:10 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:10.444 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:10 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:10.705 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:10 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:10.705 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:10 smithi203 systemd[1]: Stopping Ceph node-exporter.b for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:53:10.706 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:10 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:11.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:11 smithi203 bash[80931]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-node-exporter-b 2024-04-13T07:53:11.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:11 smithi203 bash[80990]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-node-exporter-b 2024-04-13T07:53:11.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:11 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2024-04-13T07:53:11.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:11 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@node-exporter.b.service: Failed with result 'exit-code'. 2024-04-13T07:53:11.519 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:11 smithi203 systemd[1]: Stopped Ceph node-exporter.b for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:53:11.837 INFO:journalctl@ceph.node-exporter.b.smithi203.stdout:Apr 13 07:53:11 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:11.837 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:11 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:11.838 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:11 smithi203 systemd[1]: Stopping Ceph grafana.a for ec50a262-f965-11ee-bc8d-c7b262605968... 2024-04-13T07:53:12.605 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:12 smithi203 bash[81051]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-grafana-a 2024-04-13T07:53:12.606 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:12 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@grafana.a.service: Main process exited, code=exited, status=143/n/a 2024-04-13T07:53:12.606 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:12 smithi203 bash[81110]: Error response from daemon: No such container: ceph-ec50a262-f965-11ee-bc8d-c7b262605968-grafana-a 2024-04-13T07:53:12.606 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:12 smithi203 systemd[1]: ceph-ec50a262-f965-11ee-bc8d-c7b262605968@grafana.a.service: Failed with result 'exit-code'. 2024-04-13T07:53:12.606 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:12 smithi203 systemd[1]: Stopped Ceph grafana.a for ec50a262-f965-11ee-bc8d-c7b262605968. 2024-04-13T07:53:13.019 INFO:journalctl@ceph.grafana.a.smithi203.stdout:Apr 13 07:53:12 smithi203 systemd[1]: /etc/systemd/system/ceph-ec50a262-f965-11ee-bc8d-c7b262605968@.service:23: Unit configured to use KillMode=none. This is unsafe, as it disables systemd's process lifecycle management for the service. Please update your service to use a safer KillMode=, such as 'mixed' or 'control-group'. Support for KillMode=none is deprecated and will eventually be removed. 2024-04-13T07:53:20.386 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-13T07:53:20.400 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2024-04-13T07:53:20.409 INFO:tasks.cephadm:Archiving crash dumps... 2024-04-13T07:53:20.412 DEBUG:teuthology.misc:Transferring archived files from smithi089:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941/remote/smithi089/crash 2024-04-13T07:53:20.413 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/crash -- . 2024-04-13T07:53:20.454 INFO:teuthology.orchestra.run.smithi089.stderr:tar: /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/crash: Cannot open: No such file or directory 2024-04-13T07:53:20.454 INFO:teuthology.orchestra.run.smithi089.stderr:tar: Error is not recoverable: exiting now 2024-04-13T07:53:20.456 DEBUG:teuthology.misc:Transferring archived files from smithi203:/var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/crash to /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941/remote/smithi203/crash 2024-04-13T07:53:20.456 DEBUG:teuthology.orchestra.run.smithi203:> sudo tar c -f - -C /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/crash -- . 2024-04-13T07:53:20.466 INFO:teuthology.orchestra.run.smithi203.stderr:tar: /var/lib/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/crash: Cannot open: No such file or directory 2024-04-13T07:53:20.466 INFO:teuthology.orchestra.run.smithi203.stderr:tar: Error is not recoverable: exiting now 2024-04-13T07:53:20.467 INFO:tasks.cephadm:Checking cluster log for badness... 2024-04-13T07:53:20.467 DEBUG:teuthology.orchestra.run.smithi089:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2024-04-13T07:53:20.514 INFO:teuthology.orchestra.run.smithi089.stdout:2024-04-13T07:37:56.934332+0000 mon.a (mon.0) 21 : cluster [ERR] Health check failed: 8 osds(s) are not reachable (OSD_UNREACHABLE) 2024-04-13T07:53:20.516 WARNING:tasks.cephadm:Found errors (ERR|WRN|SEC) in cluster log 2024-04-13T07:53:20.516 INFO:tasks.cephadm:Compressing logs... 2024-04-13T07:53:20.517 DEBUG:teuthology.orchestra.run.smithi089:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-13T07:53:20.560 DEBUG:teuthology.orchestra.run.smithi203:> time sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs --max-args=1 --max-procs=0 --verbose -0 --no-run-if-empty -- gzip -5 --verbose -- 2024-04-13T07:53:20.569 INFO:teuthology.orchestra.run.smithi089.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-04-13T07:53:20.573 INFO:teuthology.orchestra.run.smithi203.stderr:find: '/var/log/rbd-target-api': No such file or directory 2024-04-13T07:53:20.578 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-13T07:53:20.578 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mgr.y.log 2024-04-13T07:53:20.579 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.0.log 2024-04-13T07:53:20.579 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mgr.y.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.2.log 2024-04-13T07:53:20.580 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.0.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mds.foo.smithi089.pccnvq.log 2024-04-13T07:53:20.580 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.2.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.a.log 2024-04-13T07:53:20.585 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mds.foo.smithi089.pccnvq.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.cephadm.log 2024-04-13T07:53:20.585 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.a.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.3.log 2024-04-13T07:53:20.585 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.cephadm.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.1.log 2024-04-13T07:53:20.585 INFO:teuthology.orchestra.run.smithi089.stderr: 86.8%/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.3.log: 90.7% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mds.foo.smithi089.pccnvq.log.gz 2024-04-13T07:53:20.585 INFO:teuthology.orchestra.run.smithi089.stderr: -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.cephadm.log.gz 2024-04-13T07:53:20.585 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.log 2024-04-13T07:53:20.585 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.1.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-volume.log 2024-04-13T07:53:20.586 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/cephadm.log 2024-04-13T07:53:20.586 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.7.log 2024-04-13T07:53:20.587 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/cephadm.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.6.log 2024-04-13T07:53:20.587 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.7.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.cephadm.log 2024-04-13T07:53:20.587 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.5.log 2024-04-13T07:53:20.587 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.6.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mgr.x.log 2024-04-13T07:53:20.587 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.cephadm.log: /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.5.log: gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.log 2024-04-13T07:53:20.587 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mgr.x.log: 84.2% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.cephadm.log.gz 2024-04-13T07:53:20.587 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-volume.log 2024-04-13T07:53:20.588 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mds.foo.smithi203.sjspdy.log 2024-04-13T07:53:20.589 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.log: 92.1% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-13T07:53:20.589 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.c.log 2024-04-13T07:53:20.590 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.log: /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-volume.log: 88.3% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.log.gz 2024-04-13T07:53:20.590 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.audit.log 2024-04-13T07:53:20.590 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mds.foo.smithi203.sjspdy.log: 85.4% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mds.foo.smithi203.sjspdy.log.gz 2024-04-13T07:53:20.590 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.b.log 2024-04-13T07:53:20.591 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-volume.log: 93.4% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.log.gz 2024-04-13T07:53:20.592 INFO:teuthology.orchestra.run.smithi089.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.audit.log 2024-04-13T07:53:20.592 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.audit.log: 83.9% -- replaced with /var/log/ceph/cephadm.log.gz 2024-04-13T07:53:20.592 INFO:teuthology.orchestra.run.smithi203.stderr:gzip -5 --verbose -- /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.4.log 2024-04-13T07:53:20.593 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.b.log: 90.3% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.audit.log.gz 2024-04-13T07:53:20.602 INFO:teuthology.orchestra.run.smithi203.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.4.log: 91.3% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-volume.log.gz 2024-04-13T07:53:20.608 INFO:teuthology.orchestra.run.smithi089.stderr:/var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.c.log: /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.audit.log: 94.0% 91.1% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-volume.log.gz 2024-04-13T07:53:20.608 INFO:teuthology.orchestra.run.smithi089.stderr: -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph.audit.log.gz 2024-04-13T07:53:20.635 INFO:teuthology.orchestra.run.smithi203.stderr: 89.4% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mgr.x.log.gz 2024-04-13T07:53:20.962 INFO:teuthology.orchestra.run.smithi203.stderr: 92.6% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.b.log.gz 2024-04-13T07:53:20.998 INFO:teuthology.orchestra.run.smithi089.stderr: 89.1% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mgr.y.log.gz 2024-04-13T07:53:21.148 INFO:teuthology.orchestra.run.smithi203.stderr: 92.9% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.6.log.gz 2024-04-13T07:53:21.164 INFO:teuthology.orchestra.run.smithi089.stderr: 92.6% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.c.log.gz 2024-04-13T07:53:21.227 INFO:teuthology.orchestra.run.smithi203.stderr: 93.2% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.5.log.gz 2024-04-13T07:53:21.347 INFO:teuthology.orchestra.run.smithi203.stderr: 93.4% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.4.log.gz 2024-04-13T07:53:21.365 INFO:teuthology.orchestra.run.smithi089.stderr: 92.8% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.2.log.gz 2024-04-13T07:53:21.384 INFO:teuthology.orchestra.run.smithi089.stderr: 93.2% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.0.log.gz 2024-04-13T07:53:21.405 INFO:teuthology.orchestra.run.smithi203.stderr: 93.3% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.7.log.gz 2024-04-13T07:53:21.406 INFO:teuthology.orchestra.run.smithi203.stderr: 2024-04-13T07:53:21.407 INFO:teuthology.orchestra.run.smithi203.stderr:real 0m0.841s 2024-04-13T07:53:21.407 INFO:teuthology.orchestra.run.smithi203.stderr:user 0m3.166s 2024-04-13T07:53:21.407 INFO:teuthology.orchestra.run.smithi203.stderr:sys 0m0.106s 2024-04-13T07:53:21.453 INFO:teuthology.orchestra.run.smithi089.stderr: 93.3% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.3.log.gz 2024-04-13T07:53:21.524 INFO:teuthology.orchestra.run.smithi089.stderr: 93.5% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-osd.1.log.gz 2024-04-13T07:53:21.567 INFO:teuthology.orchestra.run.smithi089.stderr: 90.6% -- replaced with /var/log/ceph/ec50a262-f965-11ee-bc8d-c7b262605968/ceph-mon.a.log.gz 2024-04-13T07:53:21.568 INFO:teuthology.orchestra.run.smithi089.stderr: 2024-04-13T07:53:21.568 INFO:teuthology.orchestra.run.smithi089.stderr:real 0m1.006s 2024-04-13T07:53:21.568 INFO:teuthology.orchestra.run.smithi089.stderr:user 0m5.192s 2024-04-13T07:53:21.568 INFO:teuthology.orchestra.run.smithi089.stderr:sys 0m0.250s 2024-04-13T07:53:21.569 INFO:tasks.cephadm:Archiving logs... 2024-04-13T07:53:21.569 DEBUG:teuthology.misc:Transferring archived files from smithi089:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941/remote/smithi089/log 2024-04-13T07:53:21.570 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-13T07:53:22.222 DEBUG:teuthology.misc:Transferring archived files from smithi203:/var/log/ceph to /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941/remote/smithi203/log 2024-04-13T07:53:22.223 DEBUG:teuthology.orchestra.run.smithi203:> sudo tar c -f - -C /var/log/ceph -- . 2024-04-13T07:53:22.646 INFO:tasks.cephadm:Removing cluster... 2024-04-13T07:53:22.646 DEBUG:teuthology.orchestra.run.smithi089:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ec50a262-f965-11ee-bc8d-c7b262605968 --force 2024-04-13T07:53:22.764 INFO:teuthology.orchestra.run.smithi089.stdout:Deleting cluster with fsid: ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:53:24.084 DEBUG:teuthology.orchestra.run.smithi203:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid ec50a262-f965-11ee-bc8d-c7b262605968 --force 2024-04-13T07:53:24.206 INFO:teuthology.orchestra.run.smithi203.stdout:Deleting cluster with fsid: ec50a262-f965-11ee-bc8d-c7b262605968 2024-04-13T07:53:25.550 INFO:tasks.cephadm:Removing cephadm ... 2024-04-13T07:53:25.551 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-13T07:53:25.557 DEBUG:teuthology.orchestra.run.smithi203:> rm -rf /home/ubuntu/cephtest/cephadm 2024-04-13T07:53:25.562 INFO:tasks.cephadm:Teardown complete 2024-04-13T07:53:25.563 DEBUG:teuthology.run_tasks:Unwinding manager install 2024-04-13T07:53:25.574 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2024-04-13T07:53:25.575 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-13T07:53:25.600 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2024-04-13T07:53:25.668 DEBUG:teuthology.run_tasks:Unwinding manager clock 2024-04-13T07:53:25.678 INFO:teuthology.task.clock:Checking final clock skew... 2024-04-13T07:53:25.678 DEBUG:teuthology.orchestra.run.smithi089:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-13T07:53:25.681 DEBUG:teuthology.orchestra.run.smithi203:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2024-04-13T07:53:25.691 INFO:teuthology.orchestra.run.smithi089.stdout: remote refid st t when poll reach delay offset jitter 2024-04-13T07:53:25.692 INFO:teuthology.orchestra.run.smithi089.stdout:============================================================================== 2024-04-13T07:53:25.692 INFO:teuthology.orchestra.run.smithi089.stdout:*hv01.front.sepi 67.205.162.81 3 u 9 64 377 0.107 -5.709 1.275 2024-04-13T07:53:25.692 INFO:teuthology.orchestra.run.smithi089.stdout:+hv02.front.sepi 63.231.80.2 3 u 26 64 377 0.079 -3.256 0.935 2024-04-13T07:53:25.692 INFO:teuthology.orchestra.run.smithi089.stdout:+hv03.front.sepi 74.6.168.72 3 u 17 64 377 0.105 -4.920 0.939 2024-04-13T07:53:25.692 INFO:teuthology.orchestra.run.smithi089.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-04-13T07:53:25.694 INFO:teuthology.orchestra.run.smithi203.stdout: remote refid st t when poll reach delay offset jitter 2024-04-13T07:53:25.694 INFO:teuthology.orchestra.run.smithi203.stdout:============================================================================== 2024-04-13T07:53:25.695 INFO:teuthology.orchestra.run.smithi203.stdout:+hv01.front.sepi 67.205.162.81 3 u 21 64 377 0.079 -6.366 2.648 2024-04-13T07:53:25.695 INFO:teuthology.orchestra.run.smithi203.stdout:+hv02.front.sepi 63.231.80.2 3 u 13 64 377 0.079 -4.432 2.487 2024-04-13T07:53:25.695 INFO:teuthology.orchestra.run.smithi203.stdout:*hv03.front.sepi 74.6.168.72 3 u 22 64 377 0.117 -3.039 1.761 2024-04-13T07:53:25.695 INFO:teuthology.orchestra.run.smithi203.stdout: hv04.front.sepi .INIT. 16 u - 1024 0 0.000 +0.000 0.000 2024-04-13T07:53:25.695 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2024-04-13T07:53:25.706 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2024-04-13T07:53:25.706 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2024-04-13T07:53:25.732 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2024-04-13T07:53:25.765 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2024-04-13T07:53:25.834 INFO:teuthology.task.internal:Duration was 2670.809343 seconds 2024-04-13T07:53:25.835 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2024-04-13T07:53:25.868 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2024-04-13T07:53:25.868 DEBUG:teuthology.orchestra.run.smithi089:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-13T07:53:25.871 DEBUG:teuthology.orchestra.run.smithi203:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2024-04-13T07:53:25.908 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2024-04-13T07:53:25.908 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi089.front.sepia.ceph.com 2024-04-13T07:53:25.909 DEBUG:teuthology.orchestra.run.smithi089:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-13T07:53:25.956 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi203.front.sepia.ceph.com 2024-04-13T07:53:25.956 DEBUG:teuthology.orchestra.run.smithi203:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2024-04-13T07:53:25.967 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2024-04-13T07:53:25.967 DEBUG:teuthology.orchestra.run.smithi089:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-13T07:53:26.004 DEBUG:teuthology.orchestra.run.smithi203:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2024-04-13T07:53:26.065 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2024-04-13T07:53:26.066 DEBUG:teuthology.orchestra.run.smithi089:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-13T07:53:26.068 DEBUG:teuthology.orchestra.run.smithi203:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2024-04-13T07:53:26.232 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2024-04-13T07:53:26.243 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2024-04-13T07:53:26.243 DEBUG:teuthology.orchestra.run.smithi089:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-13T07:53:26.256 DEBUG:teuthology.orchestra.run.smithi203:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2024-04-13T07:53:26.269 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2024-04-13T07:53:26.282 DEBUG:teuthology.orchestra.run.smithi089:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-13T07:53:26.300 DEBUG:teuthology.orchestra.run.smithi203:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2024-04-13T07:53:26.307 INFO:teuthology.orchestra.run.smithi089.stdout:kernel.core_pattern = core 2024-04-13T07:53:26.323 INFO:teuthology.orchestra.run.smithi203.stdout:kernel.core_pattern = core 2024-04-13T07:53:26.347 DEBUG:teuthology.orchestra.run.smithi089:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-13T07:53:26.370 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-13T07:53:26.370 DEBUG:teuthology.orchestra.run.smithi203:> test -e /home/ubuntu/cephtest/archive/coredump 2024-04-13T07:53:26.395 DEBUG:teuthology.orchestra.run:got remote process result: 1 2024-04-13T07:53:26.396 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2024-04-13T07:53:26.407 INFO:teuthology.task.internal:Transferring archived files... 2024-04-13T07:53:26.408 DEBUG:teuthology.misc:Transferring archived files from smithi089:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941/remote/smithi089 2024-04-13T07:53:26.408 DEBUG:teuthology.orchestra.run.smithi089:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-13T07:53:26.441 DEBUG:teuthology.misc:Transferring archived files from smithi203:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941/remote/smithi203 2024-04-13T07:53:26.441 DEBUG:teuthology.orchestra.run.smithi203:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2024-04-13T07:53:26.465 INFO:teuthology.task.internal:Removing archive directory... 2024-04-13T07:53:26.466 DEBUG:teuthology.orchestra.run.smithi089:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-13T07:53:26.476 DEBUG:teuthology.orchestra.run.smithi203:> rm -rf -- /home/ubuntu/cephtest/archive 2024-04-13T07:53:26.510 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2024-04-13T07:53:26.522 INFO:teuthology.task.internal:Not uploading archives. 2024-04-13T07:53:26.522 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2024-04-13T07:53:26.543 INFO:teuthology.task.internal:Tidying up after the test... 2024-04-13T07:53:26.543 DEBUG:teuthology.orchestra.run.smithi089:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-13T07:53:26.546 DEBUG:teuthology.orchestra.run.smithi203:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2024-04-13T07:53:26.550 INFO:teuthology.orchestra.run.smithi089.stdout: 658622 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 13 07:53 /home/ubuntu/cephtest 2024-04-13T07:53:26.557 INFO:teuthology.orchestra.run.smithi203.stdout: 658624 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Apr 13 07:53 /home/ubuntu/cephtest 2024-04-13T07:53:26.558 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2024-04-13T07:53:26.577 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2024-04-13T07:53:26.675 INFO:teuthology.nuke:Checking targets against current locks 2024-04-13T07:53:26.702 DEBUG:teuthology.nuke:shortname: smithi089 2024-04-13T07:53:26.702 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-13T07:53:26.725 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi089.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-13 07:00:02.730171', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBAdoi9yilb7HyWm2FrHTfsEG09R0IFB6Ej94+xwRwhbx2AiVgYGiop8OAHLq4Baq+VypOHRjG6a9C7q5xjKupVU='} 2024-04-13T07:53:26.729 DEBUG:teuthology.nuke:shortname: smithi203 2024-04-13T07:53:26.729 INFO:teuthology.task.internal.check_lock:Checking locks... 2024-04-13T07:53:26.750 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi203.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2024-04-13_03:08:05-upgrade-main-distro-default-smithi/7654941', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'ubuntu', 'os_version': '22.04', 'arch': 'x86_64', 'locked': True, 'locked_since': '2024-04-13 07:00:02.728712', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBEf5PjW62a3Yg4cH5FlSOxYUIIBWGvr81e0xPm0qO3dzdkSlot2GYQ5p2a+3PExigHO9cupe0swhCx99GerfoWY='} 2024-04-13T07:53:26.780 INFO:teuthology.orchestra.console.smithi089:Power off 2024-04-13T07:53:26.780 DEBUG:teuthology.orchestra.console.smithi089:pexpect command: ipmitool -H smithi089.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-13T07:53:26.805 DEBUG:teuthology.orchestra.console.smithi089:power off output: Chassis Power Control: Down/Off 2024-04-13T07:53:26.806 DEBUG:teuthology.orchestra.console.smithi089:pexpect command: ipmitool -H smithi089.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-13T07:53:26.821 INFO:teuthology.orchestra.console.smithi203:Power off 2024-04-13T07:53:26.821 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2024-04-13T07:53:26.832 DEBUG:teuthology.orchestra.console.smithi089:check power output: Chassis Power is on 2024-04-13T07:53:26.845 DEBUG:teuthology.orchestra.console.smithi203:power off output: Chassis Power Control: Down/Off 2024-04-13T07:53:26.846 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-13T07:53:26.869 DEBUG:teuthology.orchestra.console.smithi203:check power output: Chassis Power is on 2024-04-13T07:53:30.833 DEBUG:teuthology.orchestra.console.smithi089:pexpect command: ipmitool -H smithi089.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-13T07:53:30.870 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-13T07:53:30.945 DEBUG:teuthology.orchestra.console.smithi089:check power output: Chassis Power is on 2024-04-13T07:53:30.980 DEBUG:teuthology.orchestra.console.smithi203:check power output: Chassis Power is on 2024-04-13T07:53:34.950 DEBUG:teuthology.orchestra.console.smithi089:pexpect command: ipmitool -H smithi089.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-13T07:53:34.981 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-13T07:53:35.061 DEBUG:teuthology.orchestra.console.smithi089:check power output: Chassis Power is off 2024-04-13T07:53:35.093 DEBUG:teuthology.orchestra.console.smithi203:check power output: Chassis Power is on 2024-04-13T07:53:35.162 INFO:teuthology.orchestra.console.smithi089:Power off completed 2024-04-13T07:53:39.098 DEBUG:teuthology.orchestra.console.smithi203:pexpect command: ipmitool -H smithi203.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2024-04-13T07:53:39.212 DEBUG:teuthology.orchestra.console.smithi203:check power output: Chassis Power is off 2024-04-13T07:53:39.312 INFO:teuthology.orchestra.console.smithi203:Power off completed 2024-04-13T07:53:39.417 INFO:teuthology.run:Summary data: description: upgrade/reef-x/stress-split/{0-distro/ubuntu_22.04 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/rbd-import-export mon_election/connectivity} duration: 2670.809343099594 failure_reason: timeout expired in wait_until_healthy flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=ddee5c7d8976497daf3b06d0f67fa21e status: fail success: false 2024-04-13T07:53:39.418 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2024-04-13T07:53:39.499 INFO:teuthology.run:FAIL